properly wired up the image generation api

frontend/editor2d/src/KonvaApp/Api.ts

@@ -0,0 +1,578 @@
+type NonNullableObject<T extends object> = NonNullable<T>;
+
+export interface ApiResponse<T, P = undefined> {
+  success: boolean;
+  errorMessage?: string;
+  data?: T;
+  pagination?: P;
+}
+
+export class ApiManager {
+  ApiTargets: Record<string, string> = {};
+
+  constructor() {
+    // look at the .env file
+    this.ApiTargets = {
+      BaseApi: "https://api.storyteller.ai",
+      CdnApi: "https://cdn.storyteller.ai",
+    };
+  }
+
+  public async fetch<B, T>(
+    endpoint: string,
+    {
+      method,
+      query,
+      body,
+    }: {
+      method: string;
+      query?: Record<string, string | boolean | number | undefined>;
+      body?: B;
+    },
+  ): Promise<T> {
+    const queryInString =
+      query &&
+      Object.entries(query).reduce(
+        (allOptions, [key, value]) => {
+          if (!value) {
+            return allOptions;
+          }
+          allOptions[key] = value.toString();
+          return allOptions;
+        },
+        {} as Record<string, string>,
+      );
+
+    const endpointWithQueries = queryInString
+      ? endpoint + "?" + new URLSearchParams(queryInString)
+      : endpoint;
+
+    const bodyInString = JSON.stringify(body);
+
+    const response = await fetch(endpointWithQueries, {
+      method,
+      headers: {
+        Accept: "application/json",
+        "Content-Type": "application/json",
+      },
+      credentials: "include",
+      body: bodyInString,
+    });
+    return response.json();
+  }
+
+  public async fetchMultipartFormData<T>(
+    endpoint: string,
+    {
+      method,
+      body,
+    }: {
+      method: string;
+      body: FormData;
+    },
+  ): Promise<T> {
+    const response = await fetch(endpoint, {
+      method,
+      headers: {
+        Accept: "application/json",
+      },
+      credentials: "include",
+      body: body,
+    });
+    return response.json();
+  }
+
+  protected get<T>({
+    endpoint,
+    query,
+  }: {
+    endpoint: string;
+    query?: Record<string, string | boolean | number | undefined>;
+  }): Promise<T> {
+    return this.fetch<null, T>(endpoint, { method: "GET", query });
+  }
+
+  protected post<B, T>({
+    endpoint,
+    query,
+    body,
+  }: {
+    endpoint: string;
+    query?: Record<string, string | boolean | number | undefined>;
+    body?: B;
+  }): Promise<T> {
+    return this.fetch<B, T>(endpoint, {
+      method: "POST",
+      query,
+      body,
+    });
+  }
+
+  protected delete<B, T>({
+    endpoint,
+    query,
+    body,
+  }: {
+    endpoint: string;
+    query?: Record<string, string | boolean | number | undefined>;
+    body?: B;
+  }): Promise<T> {
+    return this.fetch<B, T>(endpoint, {
+      method: "DELETE",
+      query,
+      body,
+    });
+  }
+
+  protected async postForm<T>({
+    endpoint,
+    formRecord,
+    uuid,
+    blob,
+    blobFileName,
+  }: {
+    endpoint: string;
+    formRecord: Record<string, string>;
+    uuid: string;
+    blob?: Blob | File;
+    blobFileName?: string;
+  }): Promise<T> {
+    const formData = new FormData();
+    formData.append("uuid_idempotency_token", uuid);
+    Object.entries(formRecord).forEach(([key, value]) => {
+      formData.append(key, value);
+    });
+    if (blob && blobFileName) {
+      formData.append("file", blob, blobFileName);
+    } else if (blob) {
+      formData.append("file", blob);
+    }
+
+    console.log(`HELLO${formData}`);
+    return this.fetchMultipartFormData<T>(endpoint, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+      },
+      credentials: "include",
+      body: formData,
+    });
+  }
+
+  protected camelToSnakeCase(str: string) {
+    return str.replace(/([a-z0])([A-Z])/g, "$1_$2").toLowerCase();
+  }
+
+  protected parseQueryValues(
+    params: Record<string, string | string[] | boolean | number | undefined>,
+  ): Record<string, string> {
+    return Object.entries(params).reduce(
+      (allParams, [key, value]) => {
+        if (!value) {
+          return allParams;
+        }
+        const snakeKey = this.camelToSnakeCase(key);
+        if (Array.isArray(value)) {
+          return { ...allParams, [snakeKey]: value.join(",") };
+        }
+        return { ...allParams, [snakeKey]: value.toString() };
+      },
+      {} as Record<string, string>,
+    );
+  }
+
+  protected parseBodyValues<T extends object, B extends object>(
+    params: NonNullableObject<T>,
+  ): B {
+    return Object.entries(params).reduce((allParams, [key, value]) => {
+      if (!value) {
+        return allParams;
+      }
+      const snakeKey = this.camelToSnakeCase(key);
+      if (Array.isArray(value)) {
+        return { ...allParams, [snakeKey]: value };
+      }
+      return { ...allParams, [snakeKey]: value };
+    }, {} as B);
+  }
+}
+
+export class Api extends ApiManager {
+  public async ConvertTbxToGltf({
+    mediaFileToken,
+    uuidIdempotencyToken,
+  }: {
+    mediaFileToken: string;
+    uuidIdempotencyToken: string;
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/conversion/enqueue_fbx_to_gltf`;
+
+    const body = {
+      media_file_token: mediaFileToken,
+      uuid_idempotency_token: uuidIdempotencyToken,
+    };
+
+    return this.post<
+      { media_file_token: string; uuid_idempotency_token: string },
+      { success?: boolean; inference_job_token?: string; BadInput?: string }
+    >({
+      endpoint,
+      body: body,
+    })
+      .then((response) => {
+        return {
+          success: response.success ?? false,
+          data: response.inference_job_token,
+          errorMessage: response.BadInput,
+        };
+      })
+      .catch((err) => {
+        return {
+          success: false,
+          errorMessage: err.message,
+        };
+      });
+  }
+
+  public async uploadSceneSnapshot({
+    screenshot,
+    sceneMediaToken,
+  }: {
+    screenshot: File; // base64 encoded PNG
+    sceneMediaToken?: string;
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
+    const formData = new FormData();
+    formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
+    if (sceneMediaToken) {
+      formData.append("scene_media_token", sceneMediaToken);
+    }
+
+    // for now ...
+    const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
+    formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token
+
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+      },
+      credentials: "include",
+      body: formData,
+    });
+
+    const postResponse = await response.json();
+
+    console.log(postResponse);
+
+    let result: { success: boolean; data?: string; errorMessage?: string };
+
+    if (postResponse.success) {
+      result = {
+        success: true,
+        data: postResponse.snapshot_media_token,
+        errorMessage: undefined,
+      };
+    } else {
+      result = {
+        success: false,
+        errorMessage: postResponse.BadInput,
+      };
+    }
+
+    return result;
+  }
+
+  public async enqueueImageGeneration({
+    prompt,
+    snapshotMediaToken,
+    additionalImages,
+  }: {
+    prompt: string;
+    snapshotMediaToken: string;
+    additionalImages?: string[];
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
+    // for now ...
+    const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
+    const body = {
+      uuid_idempotency_token: uuidIdempotencyToken,
+      prompt,
+      snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
+      additional_images: additionalImages,
+    };
+
+    const postResponse = await this.post<
+      {
+        uuid_idempotency_token: string;
+        prompt: string;
+        snapshot_media_token: string;
+        additional_images?: string[];
+      },
+      { success?: boolean; job_token?: string; BadInput?: string }
+    >({
+      endpoint,
+      body,
+    });
+
+    // Check if the response is successful
+    const isSuccess = postResponse.success ?? false;
+
+    // Prepare the result object
+    const result = {
+      success: isSuccess,
+      data: isSuccess ? postResponse.job_token : undefined,
+      errorMessage: isSuccess ? undefined : postResponse.BadInput,
+    };
+
+    return result;
+  }
+
+  public async pollJobSession(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`;
+
+    const response = await this.get<{
+      success?: boolean;
+      status?: string;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
+      BadInput?: string;
+    }>({
+      endpoint,
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
+  }
+
+  public async pollStudioSessionJobs(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`;
+
+    const response = await this.get<{
+      success?: boolean;
+      status?: string;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
+      BadInput?: string;
+    }>({
+      endpoint,
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
+  }
+}
+
+export class EngineApi extends ApiManager {
+  public async uploadSceneSnapshot({
+    screenshot,
+    sceneMediaToken,
+  }: {
+    screenshot: File; // base64 encoded PNG
+    sceneMediaToken?: string;
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
+    const formData = new FormData();
+    formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
+    if (sceneMediaToken) {
+      formData.append("scene_media_token", sceneMediaToken);
+    }
+
+    // for now ...
+    const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
+    formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token
+
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+      },
+      credentials: "include",
+      body: formData,
+    });
+
+    const postResponse = await response.json();
+
+    console.log(postResponse);
+
+    let result: { success: boolean; data?: string; errorMessage?: string };
+
+    if (postResponse.success) {
+      result = {
+        success: true,
+        data: postResponse.snapshot_media_token,
+        errorMessage: undefined,
+      };
+    } else {
+      result = {
+        success: false,
+        errorMessage: postResponse.BadInput,
+      };
+    }
+
+    return result;
+  }
+
+  public async enqueueImageGeneration({
+    prompt,
+    snapshotMediaToken,
+    additionalImages,
+  }: {
+    prompt: string;
+    snapshotMediaToken: string;
+    additionalImages?: string[];
+  }): Promise<ApiResponse<string>> {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
+    // for now ...
+    const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
+    const body = {
+      uuid_idempotency_token: uuidIdempotencyToken,
+      prompt,
+      snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
+      additional_images: additionalImages,
+    };
+
+    const postResponse = await this.post<
+      {
+        uuid_idempotency_token: string;
+        prompt: string;
+        snapshot_media_token: string;
+        additional_images?: string[];
+      },
+      { success?: boolean; job_token?: string; BadInput?: string }
+    >({
+      endpoint,
+      body,
+    });
+
+    // Check if the response is successful
+    const isSuccess = postResponse.success ?? false;
+
+    // Prepare the result object
+    const result = {
+      success: isSuccess,
+      data: isSuccess ? postResponse.job_token : undefined,
+      errorMessage: isSuccess ? undefined : postResponse.BadInput,
+    };
+
+    return result;
+  }
+
+  public async pollJobSession(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`;
+
+    const response = await this.get<{
+      success?: boolean;
+      status?: string;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
+      BadInput?: string;
+    }>({
+      endpoint,
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
+  }
+
+  public async pollStudioSessionJobs(jobToken: string): Promise<
+    ApiResponse<{
+      status: string;
+      result: {
+        generated_images?: string[];
+        error?: string;
+      };
+    }>
+  > {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`;
+
+    const response = await this.get<{
+      success?: boolean;
+      status?: string;
+      result?: {
+        generated_images?: string[];
+        error?: string;
+      };
+      BadInput?: string;
+    }>({
+      endpoint,
+    });
+
+    const success = response.success ?? false;
+    const status = response.status ?? "";
+    const result = response.result ?? {
+      generated_images: [],
+      error: undefined,
+    };
+    const errorMessage = response.BadInput;
+
+    return {
+      success,
+      data: { status, result },
+      errorMessage,
+    };
+  }
+}

frontend/editor2d/src/components/PromptBox.tsx

@@ -32,7 +32,7 @@ import { getCanvasRenderBitmap } from "~/signal/canvasRenderBitmap";
 import { EncodeImageBitmapToBase64 } from "~/utilities/EncodeImageBitmapToBase64";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { PopoverItem } from "~/components/reusable/Popover";
-
+import { Api, ApiManager } from "~/KonvaApp/Api";
 interface ReferenceImage {
   id: string;
   url: string;
@@ -42,8 +42,10 @@ interface ReferenceImage {
 
 export const PromptBox = () => {
   useSignals();
-  //const { lastRenderedBitmap } = useCanvasSignal();
+  const isDesktopApp = window.navigator.userAgent.includes("Tauri");
+  console.log("Is this a desktop app?", isDesktopApp);
 
+  //const { lastRenderedBitmap } = useCanvasSignal();
   const [prompt, setPrompt] = useState("");
   const [isEnqueueing, setisEnqueueing] = useState(false);
   const [useSystemPrompt, setUseSystemPrompt] = useState(true);
@@ -188,20 +190,65 @@ export const PromptBox = () => {
         referenceImages,
       );
 
-      let image = getCanvasRenderBitmap();
-
-      if (image === undefined) {
-        return;
+      if (isDesktopApp) {
+        let image = getCanvasRenderBitmap();
+        if (image === undefined) {
+          return;
+        }
+        const base64Bitmap = await EncodeImageBitmapToBase64(image);
+        const generateResponse = await invoke("image_generation_command", {
+          image: base64Bitmap,
+          prompt: prompt,
+        });
+        await new Promise((resolve) => setTimeout(resolve, 2000));
       }
 
-      const base64Bitmap = await EncodeImageBitmapToBase64(image);
+      if (!isDesktopApp) {
+        const api = new Api();
+        let image = getCanvasRenderBitmap();
+        if (image === undefined) {
+          console.error(
+            "Error: Unable to generate image. Please check the input and try again.",
+          );
+          return;
+        }
+        const base64Bitmap = await EncodeImageBitmapToBase64(image);
+        const byteString = atob(base64Bitmap);
+        const mimeString = "image/png";
+        const ab = new ArrayBuffer(byteString.length);
+        const ia = new Uint8Array(ab);
+        for (let i = 0; i < byteString.length; i++) {
+          ia[i] = byteString.charCodeAt(i);
+        }
+        const uuid = crypto.randomUUID(); // Generate a new UUID
+        const file = new File([ab], `${uuid}.png`, { type: mimeString });
 
-      const generateResponse = await invoke("image_generation_command", {
-        image: base64Bitmap,
-        prompt: prompt,
-      });
+        const snapshotMediaToken = await api.uploadSceneSnapshot({
+          screenshot: file,
+        });
 
-      await new Promise((resolve) => setTimeout(resolve, 2000));
+        if (snapshotMediaToken.data === undefined) {
+          console.error("Error: Unable to upload scene snapshot.");
+          return;
+        }
+
+        console.log("Snapshot media token:", snapshotMediaToken.data);
+
+        const response = await api.enqueueImageGeneration({
+          prompt: prompt,
+          snapshotMediaToken: snapshotMediaToken.data,
+          additionalImages: referenceImages.map((image) => image.mediaToken),
+        });
+
+        console.log("Response:", response);
+
+        if (response.success === true) {
+          console.error("Successfully enqueued image generation.");
+          return;
+        } else {
+          console.log("Image generation Not Successfull.");
+        }
+      }
     } finally {
       setisEnqueueing(false);
     }
@@ -214,10 +261,6 @@ export const PromptBox = () => {
     return iconElement.props.icon;
   };
 
-  // const handleSaveFrame = async () => {};
-
-  // const handleDownloadFrame = () => {};
-
   const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
     if (e.key === "Enter" && !e.shiftKey) {
       e.preventDefault();
@@ -360,29 +403,6 @@ export const PromptBox = () => {
             </Tooltip>
           </div>
           <div className="flex items-center gap-2">
-            {/* <Tooltip
-              content="Download frame"
-              position="top"
-              className="z-50"
-              closeOnClick={true}
-              delay={200}
-            >
-              <Button
-                className="flex h-9 items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
-                variant="secondary"
-                icon={faDownload}
-                onClick={handleDownloadFrame}
-              />
-            </Tooltip> */}
-
-            {/* <Button
-              className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
-              variant="secondary"
-              icon={faSave}
-              onClick={handleSaveFrame}
-            >
-              Save frame
-            </Button> */}
             <Button
               className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
               icon={!isEnqueueing ? faSparkles : undefined}