Merge branch 'main' into feature/add-scene

frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx

@@ -37,6 +37,7 @@ interface Props {
 const appTabs: TabItem[] = [
   { id: "2D", label: "2D" },
   { id: "3D", label: "3D" },
+  // { id: "VIDEO", label: "Video" },
 ];
 
 export const TopBar = ({
@@ -71,6 +72,44 @@ export const TopBar = ({
     }
   }, [currentAppTabId, engine3D, is3DInit]);
 
+  const handleAddToScene = async (
+    url: string,
+    media_id: string | undefined,
+  ) => {
+    console.log("add to scene", url);
+    if (currentAppTabId === "2D") {
+      console.log("Adding to 2D scene");
+
+      // from the uploaded image url.
+    } else if (currentAppTabId === "3D") {
+      console.log("Adding to 3D scene");
+
+      // from the uploaded image url.
+      // engine3D?.activeScene.loadObjectFromUrl(
+      //   "https://cdn-2.fakeyou.com/media/r/h/0/5/3/rh053gbx8yvjwk1mdvag4gkr5c0h9wbh/image_rh053gbx8yvjwk1mdvag4gkr5c0h9wbh.png",
+      // );
+
+      // from the gallery modal.
+      // engine3D?.activeScene.loadObjectFromUrl(
+      //   "https://cdn-2.fakeyou.com/media/1/t/b/0/e/1tb0e53691mdgqmtd4fmzy58wp96vwxz/image_1tb0e53691mdgqmtd4fmzy58wp96vwxz.png",
+      // );
+
+      // // working media id m_ehq8051fx4hddhja54a7zmz897cgrk
+      // engine3D?.activeScene.loadObject(
+      //   "m_ehq8051fx4hddhja54a7zmz897cgrk",
+      //   "image",
+      //   true,
+      // );
+
+      // media id from the image selected from gallery.
+      engine3D?.activeScene.loadObject(media_id, "image", true);
+    } else if (currentAppTabId === "VIDEO") {
+      console.log("Adding to Video scene");
+    } else {
+      console.warn(`Unknown tab type: ${currentAppTabId}`);
+    }
+  };
+
   return (
     <>
       <header className="fixed left-0 top-0 z-[60] w-full border-b border-white/5 bg-ui-background">
@@ -149,39 +188,7 @@ export const TopBar = ({
         activeTab={activeLibraryTab}
         onTabChange={setActiveLibraryTab}
         onDownloadClicked={downloadFileFromUrl}
-        onAddToSceneClicked={async (url) => {
-          console.log("add to scene", url);
-          // Download the image from the URL and convert to File
-          const downloadImage = async (url: string): Promise<File | null> => {
-            try {
-              // Fetch the image
-              const response = await fetch(url);
-              if (!response.ok) {
-                throw new Error(`Failed to fetch image: ${response.status}`);
-              }
-
-              // Get the blob data
-              const blob = await response.blob();
-
-              // Extract filename from URL or use a default name
-              const filename = url.split("/").pop() || "image.png";
-
-              // Create a File object from the blob
-              const file = new File([blob], filename, { type: blob.type });
-
-              return file;
-            } catch (error) {
-              console.error("Error downloading image:", error);
-              return null;
-            }
-          };
-          const file = await downloadImage(url);
-          if (file) {
-            console.log("file", file);
-          } else {
-            console.error("Failed to download image");
-          }
-        }}
+        onAddToSceneClicked={handleAddToScene}
       />
     </>
   );

frontend/apps/editor3d/app/src/pages/Page2d/Engine.ts

@@ -1,9 +1,20 @@
 import Konva from "konva";
 
 import { uiAccess, uiEvents } from "./signals";
-import { ImageNode, ShapeNode, ShapeType, TextNode, PaintNode, PreviewCopyNode } from "./KonvaApp/Nodes";
+import {
+  ImageNode,
+  ShapeNode,
+  ShapeType,
+  TextNode,
+  PaintNode,
+  PreviewCopyNode,
+} from "./KonvaApp/Nodes";
 import { UndoStackManager } from "./KonvaApp/UndoRedo";
-import { CommandManager, MatteBox, SceneManager } from "./KonvaApp/EngineUtitlities";
+import {
+  CommandManager,
+  MatteBox,
+  SceneManager,
+} from "./KonvaApp/EngineUtitlities";
 import {
   NodeIsolator,
   NodesManager,
@@ -28,9 +39,10 @@ import { invoke } from "@tauri-apps/api/core";
 
 import { DecodeBase64ToImage } from "./utilities/DecodeBase64ToImage";
 import { EncodeImageBitmapToBase64 } from "./utilities/EncodeImageBitmapToBase64";
-import { FalBackgroundRemoval, FalBackgroundRemovalRequest } from "@storyteller/tauri-api";
-
-
+import {
+  FalBackgroundRemoval,
+  FalBackgroundRemovalRequest,
+} from "@storyteller/tauri-api";
 
 export interface RenderingOptions {
   artstyle: string;
@@ -102,7 +114,7 @@ export class Engine {
     this.nodeTransformer = new NodeTransformer();
     this.uiLayer.add(this.nodeTransformer.getKonvaNode());
     // Loading Placeholders
-   
+
     // Node Isolator
     this.nodeIsolator = new NodeIsolator({
       mediaLayerRef: this.mediaLayer,
@@ -165,7 +177,7 @@ export class Engine {
       selectionManagerRef: this.selectionManager,
       renderEngineRef: this.realTimeDrawEngine,
     });
-    
+
     this.matteBox = new MatteBox({
       boardCanvasSize: {
         width: this.boardCanvasRef.clientWidth,
@@ -375,7 +387,6 @@ export class Engine {
     });
     // Listen to other toolbars
     // VideoExtraction Toolbar
-  
 
     // Listen to other requests coming from the UI
     uiEvents.onGetStagedImage((image) => {
@@ -476,7 +487,7 @@ export class Engine {
         return;
       }
 
-      console.log("Remove background success", response)
+      console.log("Remove background success", response);
 
       if ("payload" in response) {
         const base64BitmapResponse = response.payload?.base64_bytes;
@@ -741,7 +752,7 @@ export class Engine {
         this.commandManager.deleteNodes();
       }
     });
-  }   
+  }
 }
 
 // TODO: Move to utilities?

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/scene.ts

@@ -1,6 +1,7 @@
 import * as THREE from "three";
 import { GLTFLoader, GLTF } from "three/addons/loaders/GLTFLoader.js";
 import { MMDLoader } from "three/addons/loaders/MMDLoader.js";
+
 import { Font } from "three/examples/jsm/loaders/FontLoader.js";
 import { generateUUID } from "three/src/math/MathUtils.js";
 import { LoadingPlaceHolderManager } from "./placeholder_manager";
@@ -10,6 +11,9 @@ import { ChromaKeyMaterial } from "./chromakey";
 import { TimeLine } from "./timeline";
 import { ClipGroup, ClipType } from "~/enums";
 import { ClipUI } from "../clips/clip_ui";
+
+import { GetFrontendEnvironment } from "~/Classes/GetFrontendEnvironment";
+
 import { gridVisibility } from "../signals/engine";
 import { InfiniteGridHelper } from "./InfiniteGridHelper";
 import { cameras, selectedCameraId } from "../signals/camera";
@@ -423,9 +427,11 @@ class Scene {
 
   // TODO: REPLACE
   async getMediaURL(media_id: string) {
+    console.log("getMediaID!!!!!!!!!!!!!", media_id);
     const response = await this.mediaFilesApi.GetMediaFileByToken({
-      mediaFileToken: media_id
+      mediaFileToken: media_id,
     });
+    console.log("response!!!!!!!!!!!!!!", response);
     return response.data!.media_links.cdn_url;
   }
 
@@ -531,6 +537,21 @@ class Scene {
     return mmd;
   }
 
+  async loadObjectFromUrl(
+    url: string,
+    position: THREE.Vector3 = new THREE.Vector3(-0.5, 1.5, 0),
+  ): Promise<THREE.Object3D> {
+    console.log("loadObjectFromUrl!!!!!!!!", url);
+    if (
+      url.includes(".png") ||
+      url.includes(".jpg") ||
+      url.includes(".jpeg") ||
+      url.includes(".mp4")
+    ) {
+      return await this.instantiate("Image::" + url);
+    }
+  }
+
   async loadObject(
     media_id: string,
     name: string,
@@ -539,6 +560,7 @@ class Scene {
     version: number = 1.0,
   ): Promise<THREE.Object3D> {
     const url = await this.getMediaURL(media_id);
+
     if (url.includes(".pmd") || url.includes(".pmx")) {
       return await this.loadMMDWithPlaceholder(
         media_id,

frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx

@@ -8,6 +8,7 @@ import { PreviewEngineCamera } from "./comps/PreviewEngineCamera";
 import { PreviewFrameImage } from "./comps/PreviewFrameImage";
 import { pageHeight, pageWidth } from "~/signals";
 import { PoseModeSelector } from "./comps/PoseModeSelector";
+import ImageToVideo from "../PageVideo/ImageToVideo";
 import {
   timelineHeight,
   sidePanelWidth,
@@ -145,13 +146,6 @@ export const PageEditor = () => {
 
         // Force a render to update the view
         editorEngine.renderScene();
-
-        // Queue an update to ensure the engine processes the camera change
-        // Queue.publish({
-        //   queueName: QueueNames.TO_ENGINE,
-        //   action: toEngineActions.CAMERA_CHANGED,
-        //   data: selectedCamera,
-        // });
       }
 
       // Force update camera properties in the state
@@ -215,19 +209,6 @@ export const PageEditor = () => {
 
       // Force a render to update the view
       editorEngine.renderScene();
-
-      // Queue.publish({
-      //   queueName: QueueNames.TO_ENGINE,
-      //   action: toEngineActions.CAMERA_CHANGED,
-      //   data: {
-      //     id: newId,
-      //     label: `Camera ${newIndex}`,
-      //     focalLength: 24,
-      //     position: { x: randomX, y: randomY, z: randomZ },
-      //     rotation: { x: 0, y: 0, z: 0 },
-      //     lookAt: { x: 0, y: 0, z: 0 },
-      //   },
-      // });
     }
   };
 
@@ -253,6 +234,7 @@ export const PageEditor = () => {
 
   const display3d = appTabId.value === "3D" ? "block" : "none";
   const display2d = appTabId.value === "2D" ? "block" : "none";
+  const displayVideo = appTabId.value === "VIDEO" ? "block" : "none";
 
   return (
     <div className="w-screen">
@@ -343,7 +325,9 @@ export const PageEditor = () => {
                   }
                   editorEngine!.positive_prompt = prompt;
                 }}
-                snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
+                snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(
+                  editorEngine,
+                )}
               />
 
               <LoadingDots
@@ -360,9 +344,12 @@ export const PageEditor = () => {
         <KonvaRootComponent className="h-full w-full" />
       </KonvaCanvasContainer>
 
+      <div style={{ display: displayVideo }}>
+        <ImageToVideo className="h-full w-full" />
+      </div>
 
       <LoginModal
-        onClose={() => { }}
+        onClose={() => {}}
         videoSrc2D="/resources/videos/artcraft-canvas-demo.mp4"
         videoSrc3D="/resources/videos/artcraft-3d-demo.mp4"
         openAiLogo="/resources/images/openai-logo.png"

frontend/apps/editor3d/app/src/pages/PageEnigma/PageEnigma.tsx

@@ -12,7 +12,6 @@ import { useSignals } from "@preact/signals-react/runtime";
 import { useEffect, useState } from "react";
 import * as gpu from "detect-gpu";
 import { UsersApi } from "~/Classes/ApiManager";
-import { PageEnigmaComponent } from "~/pages/PageEnigma/PageEnigmaComponent";
 import { TurnOnGpu } from "~/pages/PageEnigma/TurnOnGpu";
 import { PrecisionSelector } from "./comps/PrecisionSelector/PrecisionSelector";
 import {
@@ -24,7 +23,7 @@ import {
 import { InstallSounds } from "~/pages/PageEnigma/InstallSounds";
 import { useImageGenerationFailureEvent } from "./TauriEvents/useImageGenerationFailureEvent";
 import { useImageGenerationSuccessEvent } from "./TauriEvents/useImageGenerationSuccessEvent";
-
+import { PageEditor } from "~/pages/PageEnigma/PageEditor";
 export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
   useSignals();
   useActiveJobs();
@@ -107,7 +106,7 @@ export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
   return (
     <TrackProvider>
       <EngineProvider sceneToken={sceneToken}>
-        <PageEnigmaComponent />
+        <PageEditor />
         <DragComponent />
         <PrecisionSelector
           showSignal={showPrecisionSelector}

frontend/apps/editor3d/app/src/pages/PageEnigma/PageEnigmaComponent.tsx

@@ -1,20 +1,15 @@
 import { useSignals } from "@preact/signals-react/runtime";
 import { PageEditor } from "~/pages/PageEnigma/PageEditor";
-import { PageStyling } from "~/pages/PageEnigma/PageStyling";
+
+// TODO: Remove these file
 import { currentPage } from "~/signals";
 import { Pages } from "~/pages/PageEnigma/constants/page";
 
 export const PageEnigmaComponent = () => {
   useSignals();
-
   return (
     <>
-      <div className={currentPage.value === Pages.EDIT ? "visible" : "hidden"}>
-        <PageEditor />
-      </div>
-      <div className={currentPage.value === Pages.STYLE ? "visible" : "hidden"}>
-        <PageStyling />
-      </div>
+      <PageEditor />
     </>
   );
 };

frontend/apps/editor3d/app/src/pages/PageEnigma/constants/page.ts

@@ -1,3 +1,4 @@
+// TODO: Remove this file
 export enum Pages {
   EDIT,
   STYLE,

frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx

@@ -0,0 +1,63 @@
+import { useState, useEffect, useRef } from "react";
+import { PromptBox2D } from "@storyteller/ui-promptbox";
+import { UploaderState } from "libs/common/src/lib/interfaces";
+import {
+  MaybeCanvasRenderBitmapType,
+  JobContextType,
+} from "libs/common/src/lib/types";
+
+const ImageToVideo = () => {
+  const containerRef = useRef<HTMLDivElement>(null);
+
+  const uploadImage = ({
+    title,
+    assetFile,
+    progressCallback,
+  }: {
+    title: string;
+    assetFile: File;
+    progressCallback: (newState: UploaderState) => void;
+  }): Promise<void> => {
+    throw new Error("Function not implemented.");
+  };
+
+  const getCanvasRenderBitmap = (): MaybeCanvasRenderBitmapType => {
+    throw new Error("Function not implemented.");
+  };
+
+  const EncodeImageBitmapToBase64 = (
+    imageBitmap: ImageBitmap,
+  ): Promise<string> => {
+    throw new Error("Function not implemented.");
+  };
+
+  const useJobContext = (): JobContextType => {
+    throw new Error("Function not implemented.");
+  };
+
+  return (
+    <div
+      ref={containerRef}
+      className="flex h-[calc(100vh-56px)] w-full bg-ui-background"
+    >
+      <div className="h-full w-1/2 p-4">
+        <div className="flex h-full w-full items-center justify-center rounded-md bg-gray-800">
+          <div className="text-white text-opacity-50">Image Display</div>
+          <div className="relative flex h-full w-full items-center justify-center">
+            <img
+              src=""
+              alt="Preview"
+              className="max-h-full max-w-full object-contain"
+              onError={(e) => {
+                const target = e.target as HTMLImageElement;
+                target.style.display = "none";
+              }}
+            />
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default ImageToVideo;

frontend/apps/editor3d/app/src/signals/appTab.ts

@@ -3,7 +3,7 @@ import { signal } from "@preact/signals-react";
 export const appTabId = signal("3D");
 
 export const setAppTabId = (newId: string) => {
-  if (newId != "2D" && newId != "3D") {
+  if (newId != "2D" && newId != "3D" && newId != "VIDEO") {
     console.error("Provided app ID is not valid");
     return;
   }
@@ -14,4 +14,4 @@ export const setAppTabId = (newId: string) => {
 export const is3DEditorInitialized = signal(false);
 export const setIs3DEditorInitialized = (isInitialized: boolean) => {
   is3DEditorInitialized.value = isInitialized;
-}
+};

frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx

@@ -40,7 +40,10 @@ interface GalleryModalProps {
   activeTab: string;
   onTabChange: (tabId: string) => void;
   onDownloadClicked?: (url: string) => Promise<void>;
-  onAddToSceneClicked?: (url: string) => Promise<void>;
+  onAddToSceneClicked?: (
+    url: string,
+    media_id: string | undefined
+  ) => Promise<void>;
 }
 
 export const GalleryModal = React.memo(
@@ -134,6 +137,7 @@ export const GalleryModal = React.memo(
             include_user_uploads: activeTab === "uploads",
             user_uploads_only: activeTab === "uploads",
           });
+
           if (response.success && response.data) {
             // Print the JSON response for debugging
             console.log("Media files response:", response.data);
@@ -337,12 +341,14 @@ export const GalleryModal = React.memo(
         <LightboxModal
           isOpen={isLightboxVisible}
           onClose={handleCloseLightbox}
+          onCloseGallery={onClose}
           imageUrl={imageUrl}
           imageAlt={lightboxImage?.label || ""}
           onImageError={() => imageUrl && handleImageError(imageUrl)}
           title={lightboxImage?.label}
           createdAt={lightboxImage?.createdAt}
           downloadUrl={imageUrl}
+          mediaId={lightboxImage?.id}
           onDownloadClicked={onDownloadClicked}
           onAddToSceneClicked={onAddToSceneClicked}
         />

frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx

@@ -8,6 +8,7 @@ import { faDownToLine } from "@fortawesome/pro-solid-svg-icons";
 interface LightboxModalProps {
   isOpen: boolean;
   onClose: () => void;
+  onCloseGallery: () => void;
   imageUrl?: string | null;
   imageAlt?: string;
   onImageError?: () => void;
@@ -15,20 +16,26 @@ interface LightboxModalProps {
   createdAt?: string;
   additionalInfo?: React.ReactNode;
   downloadUrl?: string;
+  mediaId?: string;
   onDownloadClicked?: (url: string) => Promise<void>;
-  onAddToSceneClicked?: (url: string) => Promise<void>;
+  onAddToSceneClicked?: (
+    url: string,
+    media_id: string | undefined
+  ) => Promise<void>;
 }
 
 export function LightboxModal({
   isOpen,
   onClose,
+  onCloseGallery,
   imageUrl,
   imageAlt = "",
   onImageError,
   title,
   createdAt,
   additionalInfo,
-  downloadUrl,
+  downloadUrl, // cdn url of the image
+  mediaId, // media id of the image
   onDownloadClicked,
   onAddToSceneClicked,
 }: LightboxModalProps) {
@@ -113,7 +120,9 @@ export function LightboxModal({
                         <Button
                           onClick={async (e) => {
                             e.stopPropagation();
-                            await onAddToSceneClicked(downloadUrl);
+                            await onAddToSceneClicked(downloadUrl, mediaId);
+                            onClose(); // close the lightbox
+                            onCloseGallery(); // close the gallery
                           }}
                         >
                           Add to Current Scene