Added reference images to the generation

enqueueImageGeneration now accepts additionalImages
enqueueImageGeneration needs testing.
Polling for jobs doesn't work,
most api calls don't work and need integration testing.

frontend/editor3d/app/Classes/ApiManager/EngineApi.ts

@@ -36,4 +36,168 @@ export class EngineApi extends ApiManager {
         };
       });
   }
+
+  public async uploadSceneSnapshot({
+    screenshot,
+    sceneMediaToken,
+  }: {
+    screenshot: string; // base64 encoded PNG
+    sceneMediaToken?: string;
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
+
+    const formData = new FormData();
+    formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
+    if (sceneMediaToken) {
+      formData.append("scene_media_token", sceneMediaToken);
+    }
+
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+      },
+      credentials: "include",
+      body: formData,
+    });
+
+    const postResponse = await response.json();
+
+    console.log(postResponse);
+
+    let result: { success: boolean; data?: string; errorMessage?: string };
+
+    if (postResponse.success) {
+      result = {
+        success: true,
+        data: postResponse.snapshot_media_token,
+        errorMessage: undefined,
+      };
+    } else {
+      result = {
+        success: false,
+        errorMessage: postResponse.BadInput,
+      };
+    }
+
+    return result;
+  }
+
+  public async enqueueImageGeneration({
+    prompt,
+    snapshotMediaToken,
+    additionalImages,
+  }: {
+    prompt: string;
+    snapshotMediaToken: string;
+    additionalImages?: string[];
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
+
+    const body = {
+      prompt,
+      snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
+      additional_images: additionalImages,
+    };
+
+    const postResponse = await this.post<
+      {
+        prompt: string;
+        snapshot_media_token: string;
+        additional_images?: string[];
+      },
+      { success?: boolean; job_token?: string; BadInput?: string }
+    >({
+      endpoint,
+      body,
+    });
+
+    // Check if the response is successful
+    const isSuccess = postResponse.success ?? false;
+
+    // Prepare the result object
+    const result = {
+      success: isSuccess,
+      data: isSuccess ? postResponse.job_token : undefined,
+      errorMessage: isSuccess ? undefined : postResponse.BadInput,
+    };
+
+    return result;
+  }
+
+  public async pollJobSession(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`;
+
+    const response = await this.get<{
+      success?: boolean;
+      status?: string;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
+      BadInput?: string;
+    }>({
+      endpoint,
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
+  }
+
+  public async pollStudioSessionJobs(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`;
+
+    const response = await this.get<{
+      success?: boolean;
+      status?: string;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
+      BadInput?: string;
+    }>({
+      endpoint,
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
+  }
 }

frontend/editor3d/app/components/reusable/UploadModalMedia/uploadImage.ts

@@ -15,6 +15,8 @@ export const uploadImage = async ({
 }) => {
   const mediaUploadApi = new MediaUploadApi();
 
+  console.log("Uploading image:", assetFile);
+
   progressCallback({ status: UploaderStates.uploadingImage });
 
   const imageResponse = await mediaUploadApi.UploadImage({
@@ -24,7 +26,10 @@ export const uploadImage = async ({
     maybe_title: "char_frame_" + title,
   });
 
+  console.log("Image response:", imageResponse);
+
   if (imageResponse == undefined) {
+    console.log("Error: Could not upload image!");
     progressCallback({
       status: UploaderStates.imageCreateError,
       errorMessage: "Could not upload image!",
@@ -33,6 +38,7 @@ export const uploadImage = async ({
   }
 
   if (!imageResponse.success || !imageResponse.data) {
+    console.log("Error:", imageResponse.errorMessage);
     progressCallback({
       status: UploaderStates.imageCreateError,
       errorMessage: imageResponse.errorMessage,
@@ -40,5 +46,9 @@ export const uploadImage = async ({
     return;
   }
 
-  progressCallback({ status: UploaderStates.success, data: imageResponse.data });
+  console.log("Upload successful:", imageResponse.data);
+  progressCallback({
+    status: UploaderStates.success,
+    data: imageResponse.data,
+  });
 };

frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts

@@ -655,6 +655,25 @@ class Editor {
     document.body.appendChild(this.stats.dom);
   }
 
+  public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
+    const snapshot = this.renderer?.domElement.toDataURL("image/png");
+    if (!snapshot) {
+      console.error("Error: Snapshot could not be created.");
+      return null;
+    }
+
+    const base64Snapshot = snapshot.split(",")[1];
+
+    if (shouldDownload) {
+      const link = document.createElement("a");
+      link.download = "scene-snapshot.png";
+      link.href = snapshot;
+      link.click();
+    }
+
+    return base64Snapshot;
+  }
+
   public async newScene(sceneTitleInput: string) {
     this.activeScene.clear();
     this.audio_engine = new AudioEngine();

frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx

@@ -1,6 +1,8 @@
 import { useState, useRef, useEffect } from "react";
 import { useSignals } from "@preact/signals-react/runtime";
-
+import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
+import { useContext } from "react";
+import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
 import {
   faPlus,
   faCamera,
@@ -18,6 +20,7 @@ import {
   faSquare,
   faRectangle,
 } from "@fortawesome/pro-regular-svg-icons";
+
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
 import { Button } from "~/components";
@@ -31,10 +34,15 @@ interface ReferenceImage {
   id: string;
   url: string;
   file: File;
+  mediaToken: string;
 }
+import { EngineApi } from "~/Classes/ApiManager/EngineApi";
+import { scene } from "~/signals/scene";
+import { UploaderStates } from "~/enums/UploaderStates";
 
 export const PromptBox = () => {
   useSignals();
+  const editorEngine = useContext(EngineContext);
   const [prompt, setPrompt] = useState("");
   const [isEnqueueing, setisEnqueueing] = useState(false);
   const [referenceImages, setReferenceImages] = useState<ReferenceImage[]>([]);
@@ -163,14 +171,25 @@ export const PromptBox = () => {
       Array.from(files).forEach((file) => {
         const reader = new FileReader();
         reader.onloadend = () => {
-          setReferenceImages((prev) => [
-            ...prev,
-            {
-              id: Math.random().toString(36).substring(7),
-              url: reader.result as string,
-              file,
+          uploadImage({
+            title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
+            assetFile: file,
+            progressCallback: (newState) => {
+              console.debug("Upload progress:", newState.data);
+              if (newState.status === UploaderStates.success && newState.data) {
+                const referenceImage: ReferenceImage = {
+                  id: Math.random().toString(36).substring(7),
+                  url: reader.result as string,
+                  file,
+                  mediaToken: newState.data || "",
+                };
+                setReferenceImages((prev) => [...prev, referenceImage]);
+                console.log("Reference image added:", referenceImage);
+              } else {
+                console.error("No data from uploadImage");
+              }
             },
-          ]);
+          });
         };
         reader.readAsDataURL(file);
       });
@@ -218,18 +237,36 @@ export const PromptBox = () => {
   const handleEnqueue = async () => {
     if (!prompt.trim()) return;
 
-    setisEnqueueing(true);
-    try {
-      // Here we would pass both the prompt and reference images to the generation
-      console.log(
-        "Enqueuing with prompt:",
-        prompt,
-        "and reference images:",
-        referenceImages,
-      );
-      await new Promise((resolve) => setTimeout(resolve, 2000));
-    } finally {
-      setisEnqueueing(false);
+    if (editorEngine) {
+      editorEngine.positive_prompt = prompt;
+
+      const engineApi = new EngineApi();
+      const snapshot = editorEngine.snapShotOfCurrentFrame(false);
+
+      const snapshotResult = await engineApi.uploadSceneSnapshot({
+        screenshot: snapshot || "",
+        sceneMediaToken: "",
+      });
+
+      await engineApi.enqueueImageGeneration({
+        prompt: prompt,
+        snapshotMediaToken: snapshotResult.data || "",
+        additionalImages: referenceImages.map((image) => image.mediaToken),
+      });
+
+      setisEnqueueing(true);
+      try {
+        // Here we would pass both the prompt and reference images to the generation
+        console.log(
+          "Enqueuing with prompt:",
+          prompt,
+          "and reference images:",
+          referenceImages,
+        );
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+      } finally {
+        setisEnqueueing(false);
+      }
     }
   };
 
@@ -258,6 +295,24 @@ export const PromptBox = () => {
     }
   };
 
+  const handleSaveFrame = async () => {
+    if (editorEngine) {
+      const snapshot = editorEngine.snapShotOfCurrentFrame(false);
+      const engineApi = new EngineApi();
+      const result = await engineApi.uploadSceneSnapshot({
+        screenshot: snapshot || "",
+        sceneMediaToken: "",
+      });
+      console.log(result);
+    }
+  };
+
+  const handleDownloadFrame = () => {
+    if (editorEngine) {
+      editorEngine.snapShotOfCurrentFrame(true);
+    }
+  };
+
   return (
     <div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 flex-col gap-3">
       {referenceImages.length > 0 && (
@@ -360,6 +415,7 @@ export const PromptBox = () => {
               className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
               variant="secondary"
               icon={faDownload}
+              onClick={handleDownloadFrame}
             >
               Download frame
             </Button>
@@ -367,6 +423,7 @@ export const PromptBox = () => {
               className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
               variant="secondary"
               icon={faSave}
+              onClick={handleSaveFrame}
             >
               Save frame
             </Button>