Display a checkerboard pattern for transparency

frontend/apps/editor3d/app/src/Helpers/CanvasHelpers.ts

@@ -1,12 +1,12 @@
 
-export const normalizeCanvas = (canvas: HTMLCanvasElement, width: number, height: number): HTMLCanvasElement => {
-  const newCanvas = document.createElement("canvas");
+export const normalizeCanvas = (canvas: HTMLCanvasElement, width: number, height: number): OffscreenCanvas => {
+  const newCanvas = new OffscreenCanvas(width, height);
   newCanvas.width = width;
   newCanvas.height = height;
 
   const ctx = newCanvas.getContext("2d");
   if (!ctx) {
-    throw new Error("Failed to get canvas context");
+    throw new Error("Failed to get offscreen canvas context");
   }
 
   ctx.imageSmoothingEnabled = true;

frontend/apps/editor3d/app/src/pages/PageEdit/EditPaintSurface.tsx

@@ -20,6 +20,8 @@ import { useGlobalMouseUp } from "../PageDraw/hooks/useGlobalMouseUp";
 import { useRightPanelLayoutManagement } from "../PageDraw/hooks/useRightPanelLayoutManagement";
 import { LineNode } from "../PageDraw/stores/SceneState";
 import { Node } from "../PageDraw/Node";
+import { checkerboard } from "@storyteller/common";
+import { loadImageFromUrl } from "~/Helpers/ImageHelpers";
 
 export type EditPaintSurfaceProps = {
   nodes: Node[];
@@ -58,6 +60,9 @@ export const EditPaintSurface = ({
   const [snapshotImage, setSnapshotImage] = useState<HTMLImageElement | null>(
     null,
   );
+  // TODO: Polish this by using canvas graphics instead of a bitmap
+  // Or at least manually do the pattern fill to avoid image interpolation
+  const [checkerImage, setCheckerImage] = useState<HTMLImageElement | null>(null);
   const rightContainerRef = React.useRef<HTMLDivElement>(null);
   const cursorLayerRef = React.useRef<Konva.Layer>(null);
   const cursorShapeRef = React.useRef<Konva.Circle>(null);
@@ -975,6 +980,11 @@ export const EditPaintSurface = ({
     }
   }, [selectedNodeIds, stageRef, transformerRefs]);
 
+  // Load the checkboard image for transparency background
+  useEffect(() => {
+    loadImageFromUrl(checkerboard).then(setCheckerImage);
+  }, []);
+
   return (
     <SplitPane
       singlePaneMode={singlePaneMode}
@@ -1002,6 +1012,26 @@ export const EditPaintSurface = ({
             onMouseEnter={handleStageMouseEnter}
             onMouseLeave={handleStageMouseLeave}
           >
+            <Layer
+              clipFunc={(ctx) => {
+                ctx.rect(0, 0, store.getAspectRatioDimensions().width, store.getAspectRatioDimensions().height); // leftPanelWidth, leftPanelHeight);
+              }}
+              imageSmoothingEnabled={false} // Disable image smoothing for pixel art
+            >
+              <Rect
+                x={0}
+                y={0}
+                fillPatternImage={checkerImage || undefined}
+                fillPatternRepeat="repeat"
+                fillPatternScaleX={window.devicePixelRatio}
+                fillPatternScaleY={window.devicePixelRatio}
+                width={store.getAspectRatioDimensions().width}
+                height={store.getAspectRatioDimensions().height}
+                listening={false}
+                zIndex={-2}
+                imageSmoothingEnabled={false} // Disable image smoothing for pixel art
+              />
+            </Layer>
             <Layer
               clipFunc={(ctx) => {
                 ctx.rect(0, 0, store.getAspectRatioDimensions().width, store.getAspectRatioDimensions().height); // leftPanelWidth, leftPanelHeight);

frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx

@@ -38,9 +38,9 @@ const PageEdit = () => {
   // Create refs for stage and image
   const stageRef = useRef<Konva.Stage>({} as Konva.Stage);
   const leftPanelRef = useRef<Konva.Layer>({} as Konva.Layer);
-  const rectRef = useRef<Konva.Rect>({} as Konva.Rect);
+  const baseImageKonvaRef = useRef<Konva.Image>({} as Konva.Image);
   const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({});
-  const [baseImage, setBaseImage] = useState<BaseSelectorImage | null>(null);
+  const [isEnqueuing, setIsEnqueuing] = useState<boolean>(false);
 
   // Use the Zustand store
   const store = useEditStore();
@@ -115,29 +115,6 @@ const PageEdit = () => {
     };
   }, [store]);
 
-  const handleImageUpload = (files: File[]): void => {
-    // Place images at center of viewport with offset for multiple images
-    const centerX = 512; // leftPanelWidth / 2
-    const centerY = 512; // leftPanelHeight / 2
-
-    console.log("Image upload started with files:", files);
-    console.log("Center coordinates:", { centerX, centerY });
-
-    files.forEach((file, index) => {
-      console.log(`Processing file ${index}:`, file.name);
-
-      store.createImageFromFile(
-        centerX + index * 60, // Offset each image
-        centerY + index * 60,
-        file,
-      );
-      console.log(`Created image at position:`, {
-        x: centerX + index * 60,
-        y: centerY + index * 60,
-      });
-    });
-  };
-
   const modelList: PopoverItem[] = [
     {
       label: "GPT-4o",
@@ -159,17 +136,6 @@ const PageEdit = () => {
     setSelectedModel(item.label);
   };
 
-  const onEnqueuedPressed = async () => {
-    // takes snap shot and then a global variable in the engine will invoke the inference.
-    const image = await captureStageImageBitmap(stageRef, transformerRefs);
-    if (!image) {
-      console.error("Failed to capture stage image");
-      return;
-    } else {
-      setCanvasRenderBitmap(image);
-    }
-  };
-
   const onFitPressed = async () => {
     // Get the stage and its container dimensions
     const stage = stageRef.current;
@@ -190,16 +156,16 @@ const PageEdit = () => {
   }
 
   // Create a function to use the left layer ref and download the bitmap from it
-  const downloadLeftPanelBitmap = async(): Promise<Uint8Array> => {
-    if (!stageRef.current || !leftPanelRef.current || !rectRef.current) {
+  const getMaskArrayBuffer = async (): Promise<Uint8Array> => {
+    if (!stageRef.current || !leftPanelRef.current || !baseImageKonvaRef.current) {
       console.error("Stage or left panel ref is not available");
-      return;
+      throw new Error("Stage or left panel or base image ref is not available");
     }
 
     const layer = leftPanelRef.current;
 
     // Get the canvas area that's covered by the image/rectangle
-    const rect = rectRef.current;
+    const rect = baseImageKonvaRef.current;
     const layerCrop = layer.toCanvas({
       x: stageRef.current.x(),
       y: stageRef.current.y(),
@@ -208,46 +174,24 @@ const PageEdit = () => {
       pixelRatio: 1 / stageRef.current.scaleX(),
     });
 
-    // // Using the pixelRatio scaling may result in off-by-one rounding errors,
-    // // So we re-fit the image to a canvas of precise size.
-    // const fittedCanvas = normalizeCanvas(layerCrop, rect.width(), rect.height());
-    // const downloadCallback = (blob: Blob | null) => {
-    //   if (!blob) {
-    //     console.error("Failed to create blob from canvas");
-    //     return;
-    //   }
-    //   const url = URL.createObjectURL(blob);
-    //   const link = document.createElement("a");
-    //   link.href = url;
-    //   link.download = "artcraft_snapshot.png";
-    //   link.click();
-    //   console.log('downloadCallback', url);
-    // }
-    // fittedCanvas.toBlob(downloadCallback, "image/png", 1.0);
+    // Using the pixelRatio scaling may result in off-by-one rounding errors,
+    // So we re-fit the image to a canvas of precise size.
+    const fittedCanvas = normalizeCanvas(layerCrop, rect.width(), rect.height());
 
-    const normalizeCanvas2 = (canvas: HTMLCanvasElement, width: number, height: number): OffscreenCanvas => {
-      const newCanvas = new OffscreenCanvas(width, height);
-
-      const ctx = newCanvas.getContext("2d");
-      if (!ctx) {
-        throw new Error("Failed to get canvas context");
-      }
-
-      ctx.imageSmoothingEnabled = true;
-      ctx.drawImage(canvas, 0, 0, width, height);
-      return newCanvas;
-    }
-
-    const fittedCanvas2 = normalizeCanvas2(layerCrop, rect.width(), rect.height());
-
-    const blob = await fittedCanvas2.convertToBlob({ type: 'image/png' });
+    const blob = await fittedCanvas.convertToBlob({ type: 'image/png' });
     const arrayBuffer = await blob.arrayBuffer();
-  
-    return new Uint8Array(arrayBuffer);
 
+    return new Uint8Array(arrayBuffer);
   };
 
   const handleGenerate = async (prompt: string) => {
+    if (isEnqueuing) {
+      console.warn("Already enqueuing an image, please wait.");
+      return;
+    }
+
+    setIsEnqueuing(true);
+
     const editedImageToken = store.baseImageInfo?.mediaToken;
 
     if (!editedImageToken) {
@@ -256,7 +200,7 @@ const PageEdit = () => {
     }
 
     // TODO: Call inference API here
-    let arrayBuffer = await downloadLeftPanelBitmap();
+    const arrayBuffer = await getMaskArrayBuffer();
 
     const response = await EnqueueImageInpaint({
       model: EnqueueImageInpaintModel.FluxPro1,
@@ -265,6 +209,8 @@ const PageEdit = () => {
       prompt: prompt,
       image_count: 1,
     });
+
+    setIsEnqueuing(false);
   };
 
   // Display image selector on launch, otherwise hide it
@@ -304,6 +250,7 @@ const PageEdit = () => {
           onModeChange={(mode: string) => { store.setActiveTool(mode as ActiveEditTool) }}
           selectedMode={store.activeTool}
           onGenerateClick={handleGenerate}
+          isDisabled={isEnqueuing}
         />
       </div>
       <div className="relative z-0">
@@ -356,7 +303,7 @@ const PageEdit = () => {
             stageRef={stageRef}
             transformerRefs={transformerRefs}
             leftPanelRef={leftPanelRef}
-            baseImageRef={rectRef}
+            baseImageRef={baseImageKonvaRef}
           />
         </ContextMenuContainer>
       </div>

frontend/apps/editor3d/app/src/pages/PageEdit/PromptEditor/PromptEditor.tsx

@@ -1,25 +1,20 @@
 import React from "react";
 import { PromptBoxEdit, PromptBoxEditProps } from "@storyteller/ui-promptbox";
 
-// Set this value on when enqueue is pressed nasty global variable.
-import { JobProvider } from "~/pages/PageDraw/JobContext";
-
 const PromptEditor: React.FC<PromptBoxEditProps> = ({
   onModeChange,
   selectedMode,
   onGenerateClick,
+  isDisabled
 }) => {
   return (
     <div className="mx-auto flex w-full max-w-3xl flex-col space-y-2">
-
-      <JobProvider>
-        <PromptBoxEdit
-          onModeChange={onModeChange}
-          selectedMode={selectedMode}
-          onGenerateClick={onGenerateClick}
-          isDisabled={false}
-        />
-      </JobProvider>
+      <PromptBoxEdit
+        onModeChange={onModeChange}
+        selectedMode={selectedMode}
+        onGenerateClick={onGenerateClick}
+        isDisabled={isDisabled}
+      />
     </div>
   );
 };

frontend/libs/common/src/index.ts

@@ -1,3 +1,4 @@
+export * from "./lib/assets";
 export * from "./lib/enums";
 export * from "./lib/interfaces";
 export * from "./lib/types";

frontend/libs/common/src/lib/assets/images/index.ts

@@ -0,0 +1,3 @@
+import checkerboard from './checkerboard.png';
+
+export { checkerboard };

frontend/libs/common/src/lib/assets/index.ts

@@ -0,0 +1 @@
+export * from "./images";

frontend/libs/components/promptbox/src/lib/PromptBoxEdit.tsx

@@ -8,7 +8,6 @@ import {
   faSpinnerThird
 } from "@fortawesome/pro-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
-import { useSignals } from "@preact/signals-react/runtime";
 import { Button, ToggleButton } from "@storyteller/ui-button";
 import { ButtonIconSelect } from "@storyteller/ui-button-icon-select";
 import { Tooltip } from "@storyteller/ui-tooltip";
@@ -27,7 +26,6 @@ export const PromptBoxEdit = ({
   onGenerateClick,
   isDisabled
 }: PromptBoxEditProps) => {
-  useSignals();
 
   const [prompt, setPrompt] = useState("");
   const [useSystemPrompt, setUseSystemPrompt] = useState(true);