Make 3D re-mount

frontend/apps/editor3d/app/src/components/signaled/TopBar/TopBar.tsx

@@ -69,10 +69,10 @@ export const TopBar = ({
   pageName,
   appTabIdSignal,
   setAppTabId,
-  is3DInitSignal,
 }: Props) => {
   useSignals();
 
+  const editorEngine = useContext(EngineContext);
   const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
 
   const [url, setUrl] = useState<string>("");
@@ -80,19 +80,10 @@ export const TopBar = ({
 
   const engine3D = useContext(EngineContext);
   const handleTabChange = (tabId: string) => {
-    if (appTabIdSignal.peek() == "3D") {
-      engine3D?.unmountEngine();
-    }
     setAppTabId(tabId);
   };
 
   const currentAppTabId = appTabIdSignal.value;
-  const is3DInit = is3DInitSignal.value;
-  useEffect(() => {
-    if (currentAppTabId == "3D" && is3DInit) {
-      engine3D?.remountEngine();
-    }
-  }, [currentAppTabId, engine3D, is3DInit]);
 
   const handleAddToScene = async (
     url: string,

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts

@@ -53,7 +53,7 @@ import FindSurfaces from "./FindSurfaces.js";
 import Stats from "three/examples/jsm/libs/stats.module.js";
 import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
 import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
-import { is3DEditorInitialized, setIs3DEditorInitialized } from "~/signals/appTab";
+import { is3DEditorInitialized, is3DPageMounted, setIs3DEditorInitialized } from "~/signals/appTab";
 
 export type EditorInitializeConfig = {
   sceneToken: string;
@@ -131,6 +131,7 @@ class Editor {
   // getSceneSignals: () => SceneSignal;
   render_width: number;
   render_height: number;
+  containerResizeObserver: ResizeObserver | undefined;
 
   positive_prompt: string;
   negative_prompt: string;
@@ -323,7 +324,7 @@ class Editor {
 
 
     this.renderIndex = 0;
-  
+
   }
 
   // Add helper method to convert focal length to FOV
@@ -388,6 +389,8 @@ class Editor {
   }
   updateSceneContainer(newContainer: HTMLDivElement) {
     this.container = newContainer;
+    this.containerResizeObserver?.disconnect()
+    this.containerResizeObserver?.observe(this.container);
   }
   engineCanvasMayReset() {
     //TODO: we should not need this, if the this canvas is reset,
@@ -403,6 +406,9 @@ class Editor {
   }
   updateEngineCanvas(newCanvas: HTMLCanvasElement) {
     this.canvReference = newCanvas;
+    if (this.renderer) {
+      this.renderer.domElement = this.canvReference;
+    }
   }
   camViewCanvasMayReset() {
     //TODO: we should not need this, if the this canvas is reset,
@@ -418,6 +424,9 @@ class Editor {
   }
   updateCamViewCanvas(newCanvas: HTMLCanvasElement) {
     this.canvasRenderCamReference = newCanvas;
+    if (this.rawRenderer) {
+      this.rawRenderer.domElement = this.canvasRenderCamReference;
+    }
   }
 
   changeRenderCameraAspectRatio(newAspectRatio: CameraAspectRatio) {
@@ -444,8 +453,8 @@ class Editor {
     sceneContainerEl,
   }: EditorInitializeConfig) {
     if (!this.can_initialize) {
-      console.log("Editor Already Initialized");
-      return;
+      console.log("Reinitializing 3D editor");
+      // return;
     }
 
     this.can_initialize = false;
@@ -642,10 +651,6 @@ class Editor {
     // Attach freecam state controller
     this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
 
-    // This will enable all event and render loops
-    // We'll disable it here so the UI events can control is manually
-    // this.remountEngine();
-
     if (!this.utils.isEmpty(sceneToken)) {
       this.loadScene(sceneToken);
     } else {
@@ -666,6 +671,10 @@ class Editor {
     loadingBarIsShowing.value = false;
 
     setIs3DEditorInitialized(true);
+
+    // This will enable all event and render loops
+    // We'll disable it here so the UI events can control is manually
+    this.remountEngine();
   }
 
   public isMovable(): boolean {
@@ -1241,7 +1250,7 @@ class Editor {
           console.log(`COLLECTING COLOR FRAMES COUNTED: ${this.renderIndex}`);
           await this.engineFrameBuffers.collectColorFrames(this.renderIndex);
 
-         
+
           this.renderIndex = 0;
           console.time("Stop Playback And Upload Video Time");
           await this.stopPlaybackAndUploadVideo();
@@ -1252,7 +1261,7 @@ class Editor {
         } catch (error) {
           // don't use cache in this case.
           this.processingHasFailed = true;
-        
+
           console.log(`Video Generation: ${error}`);
         }
       } // End Timeline Playing
@@ -1433,10 +1442,17 @@ class Editor {
       return;
     }
 
+    if (!is3DPageMounted.peek()) {
+      console.log("3D mount: Wait for DOM mount");
+      return;
+    }
+
+    this.isMounted = true;
     this.startRenderLoop();
     this.sceneManager?.attachEventListeners();
     this.enableFreeCamControls();
     this.cameraViewControls?.attachEventListeners();
+    console.log("3D Editor Engine remounted");
   }
 
   unmountEngine() {
@@ -1444,6 +1460,8 @@ class Editor {
     this.sceneManager?.detachEventListeners();
     this.disableFreeCamControls();
     this.cameraViewControls?.detachEventListeners();
+    this.isMounted = false;
+    console.log("3D Editor Engine unmounted");
   }
 
   change_mode(type: "translate" | "rotate" | "scale") {
@@ -1693,7 +1711,7 @@ class Editor {
       return;
     }
 
-    const resizeObserver = new ResizeObserver((entries) => {
+    this.containerResizeObserver = new ResizeObserver((entries) => {
       for (const entry of entries) {
         const { width, height } = entry.contentRect;
         if (this.camera) {
@@ -1705,7 +1723,7 @@ class Editor {
       }
     });
 
-    resizeObserver.observe(this.container);
+    this.containerResizeObserver.observe(this.container);
   }
 
   getAssetType(selected: THREE.Object3D<THREE.Object3DEventMap>): AssetType {

frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx

@@ -1,4 +1,4 @@
-import React, { useContext, useEffect } from "react";
+import React, { useContext, useEffect, useRef } from "react";
 import { useSignals } from "@preact/signals-react/runtime";
 import { TopBar } from "~/components";
 import { Controls3D } from "./comps/Controls3D";
@@ -31,7 +31,7 @@ import { PopoverItem } from "@storyteller/ui-popover";
 import { LoadingDots } from "@storyteller/ui-loading";
 import { OnboardingHelper } from "./comps/OnboardingHelper";
 import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDisplay";
-import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
+import { appTabId, is3DEditorInitialized, is3DPageMounted, set3DPageMounted, setAppTabId } from "~/signals/appTab";
 import { KonvaCanvasContainer } from "../Page2d/KonvaCanvasContainer";
 import { KonvaRootComponent } from "../Page2d/KonvaRootComponent";
 import {
@@ -329,10 +329,21 @@ export const PageEditor = () => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [appTabId.value, editorEngine]);
 
-  const display3d = appTabId.value === "3D" ? "block" : "none";
-  const display2d = appTabId.value === "2D" ? "block" : "none";
-  const displayVideo = appTabId.value === "VIDEO" ? "block" : "none";
-  const displayImage = appTabId.value === "IMAGE" ? "block" : "none";
+  const display3d = appTabId.value === "3D";
+  const display2d = appTabId.value === "2D";
+  const displayVideo = appTabId.value === "VIDEO";
+  const displayImage = appTabId.value === "IMAGE";
+
+  const is3DInit = is3DEditorInitialized.value;
+  const currentAppTabId = appTabId.value;
+  useEffect(() => {
+    if (currentAppTabId !== "3D") {
+      set3DPageMounted(false);
+      return;
+    }
+
+    set3DPageMounted(true);
+  }, [currentAppTabId, editorEngine, is3DInit])
 
   return (
     <div className="w-screen">
@@ -342,119 +353,128 @@ export const PageEditor = () => {
         setAppTabId={setAppTabId}
         is3DInitSignal={is3DEditorInitialized}
       />
-      <>
-        <OnboardingHelper />
-        <div
-          className="relative flex w-screen"
-          style={{ height: "calc(100vh - 68px)", display: display3d }}
-        >
-          {/* Engine section/side panel */}
+      {display3d && (
+        <>
+          <OnboardingHelper />
           <div
-            id="engine-n-panels-wrapper"
-            className="flex"
-            style={{
-              height,
-            }}
+            className="relative flex w-screen"
+            style={{ height: "calc(100vh - 68px)" }}
           >
-            <div className="relative w-full overflow-hidden bg-transparent">
-              <SceneContainer>
-                <EditorCanvas />
-                <PreviewFrameImage />
-              </SceneContainer>
+            {/* Engine section/side panel */}
+            <div
+              id="engine-n-panels-wrapper"
+              className="flex"
+              style={{
+                height,
+              }}
+            >
+              <div className="relative w-full overflow-hidden bg-transparent">
+                <SceneContainer>
+                  <EditorCanvas />
+                  <PreviewFrameImage />
+                </SceneContainer>
 
-              {/* Focal Length Display */}
-              <FocalLengthDisplay />
+                {/* Focal Length Display */}
+                <FocalLengthDisplay />
 
-              {/* Pose Mode Selector */}
-              <PoseModeSelector />
+                {/* Pose Mode Selector */}
+                <PoseModeSelector />
 
-              {/* Top controls */}
-              <div
-                className="absolute left-0 top-0 w-full"
-                onClick={handleOverlayClick}
-              >
-                <div className="grid grid-cols-3 gap-4">
-                  <ControlsTopButtons />
-                  <Controls3D />
-                </div>
-              </div>
-
-              {/* Bottom controls */}
-              <div
-                className="absolute bottom-0 left-0"
-                style={{
-                  width: pageWidth.value,
-                }}
-                onClick={handleOverlayClick}
-              >
+                {/* Top controls */}
                 <div
-                  className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
-                  style={{ transform: `scale(${getScale()})` }}
+                  className="absolute left-0 top-0 w-full"
+                  onClick={handleOverlayClick}
                 >
-                  <Outliner />
-                  <PreviewEngineCamera />
+                  <div className="grid grid-cols-3 gap-4">
+                    <ControlsTopButtons />
+                    <Controls3D />
+                  </div>
                 </div>
 
-                <ControlPanelSceneObject />
+                {/* Bottom controls */}
+                <div
+                  className="absolute bottom-0 left-0"
+                  style={{
+                    width: pageWidth.value,
+                  }}
+                  onClick={handleOverlayClick}
+                >
+                  <div
+                    className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
+                    style={{ transform: `scale(${getScale()})` }}
+                  >
+                    <Outliner />
+                    <PreviewEngineCamera />
+                  </div>
+
+                  <ControlPanelSceneObject />
+                </div>
+
+                <PromptBox3D
+                  cameras={cameras}
+                  cameraAspectRatio={cameraAspectRatio}
+                  disableHotkeyInput={disableHotkeyInput}
+                  enableHotkeyInput={enableHotkeyInput}
+                  gridVisibility={gridVisibility}
+                  setGridVisibility={setGridVisibility}
+                  selectedCameraId={selectedCameraId}
+                  deleteCamera={deleteCamera}
+                  focalLengthDragging={focalLengthDragging}
+                  isPromptBoxFocused={isPromptBoxFocused}
+                  uploadImage={uploadImage}
+                  handleCameraSelect={handleCameraSelect}
+                  handleAddCamera={handleAddCamera}
+                  handleCameraNameChange={handleCameraNameChange}
+                  handleCameraFocalLengthChange={handleCameraFocalLengthChange}
+                  onAspectRatioSelect={onAspectRatioSelect}
+                  setEnginePrompt={(prompt) => {
+                    console.log("setEnginePrompt", prompt);
+                    if (!editorEngine) {
+                      console.log("editorEngine is not available");
+                      return;
+                    }
+                    editorEngine!.positive_prompt = prompt;
+                  }}
+                  snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(
+                    editorEngine,
+                  )}
+                />
+
+                <LoadingDots
+                  className="absolute left-0 top-0 z-50"
+                  isShowing={editorLoader.value.isShowing}
+                  type="bricks"
+                  message={editorLoader.value.message}
+                />
               </div>
-
-              <PromptBox3D
-                cameras={cameras}
-                cameraAspectRatio={cameraAspectRatio}
-                disableHotkeyInput={disableHotkeyInput}
-                enableHotkeyInput={enableHotkeyInput}
-                gridVisibility={gridVisibility}
-                setGridVisibility={setGridVisibility}
-                selectedCameraId={selectedCameraId}
-                deleteCamera={deleteCamera}
-                focalLengthDragging={focalLengthDragging}
-                isPromptBoxFocused={isPromptBoxFocused}
-                uploadImage={uploadImage}
-                handleCameraSelect={handleCameraSelect}
-                handleAddCamera={handleAddCamera}
-                handleCameraNameChange={handleCameraNameChange}
-                handleCameraFocalLengthChange={handleCameraFocalLengthChange}
-                onAspectRatioSelect={onAspectRatioSelect}
-                setEnginePrompt={(prompt) => {
-                  console.log("setEnginePrompt", prompt);
-                  if (!editorEngine) {
-                    console.log("editorEngine is not available");
-                    return;
-                  }
-                  editorEngine!.positive_prompt = prompt;
-                }}
-                snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(
-                  editorEngine,
-                )}
-              />
-
-              <LoadingDots
-                className="absolute left-0 top-0 z-50"
-                isShowing={editorLoader.value.isShowing}
-                type="bricks"
-                message={editorLoader.value.message}
-              />
             </div>
           </div>
+        </>
+      )}
+
+      {display2d && (
+        <KonvaCanvasContainer>
+          <KonvaRootComponent className="h-full w-full" />
+        </KonvaCanvasContainer>
+      )}
+
+      {displayVideo && (
+        <div>
+          <ImageToVideo
+            imageMediaId={topNavMediaId.value}
+            imageUrl={topNavMediaUrl.value}
+          />
         </div>
-      </>
-      <KonvaCanvasContainer style={{ display: display2d }}>
-        <KonvaRootComponent className="h-full w-full" />
-      </KonvaCanvasContainer>
+      )}
 
-      <div style={{ display: displayVideo }}>
-        <ImageToVideo
-          imageMediaId={topNavMediaId.value}
-          imageUrl={topNavMediaUrl.value}
-        />
-      </div>
-
-      <div style={{ display: displayImage }}>
-        <TextToImage
-          imageMediaId={topNavMediaId.value}
-          imageUrl={topNavMediaUrl.value}
-        />
-      </div>
+      {displayImage && (
+        <div>
+          <TextToImage
+            imageMediaId={topNavMediaId.value}
+            imageUrl={topNavMediaUrl.value}
+          />
+        </div>
+      )}
 
       <LoginModal
         onClose={() => { }}

frontend/apps/editor3d/app/src/pages/PageEnigma/comps/EngineCanvases.tsx

@@ -1,20 +1,14 @@
 import { useCallback, useContext } from "react";
-import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
+import { camViewCanvasSignal, editorCanvasSignal, EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
 
 export const EditorCanvas = () => {
-  const editorEngine = useContext(EngineContext);
-
   const canvasCallbackRef = useCallback(
     (node: HTMLCanvasElement) => {
-      if (node && editorEngine) {
-        if (!editorEngine.canvReference) {
-          editorEngine.setEditorCanvas(node);
-        } else {
-          editorEngine.updateEngineCanvas(node);
-        }
+      if (node) {
+        editorCanvasSignal.value = node;
       }
     },
-    [editorEngine],
+    [],
   );
 
   return (
@@ -28,22 +22,16 @@ export const EditorCanvas = () => {
 };
 
 export const CameraViewCanvas = ({ className }: { className?: string }) => {
-  const editorEngine = useContext(EngineContext);
   const canvasCallbackRef = useCallback(
     (node: HTMLCanvasElement) => {
-      if (node && editorEngine) {
-        if (!editorEngine.canvasRenderCamReference) {
-          editorEngine.setCamViewCanvas(node);
-        } else {
-          editorEngine.updateCamViewCanvas(node);
-        }
+      if (node) {
+        camViewCanvasSignal.value = node;
       }
-      //else just do nothing
     },
-    [editorEngine],
+    [],
   );
 
   return (
-    <canvas className={className} ref={canvasCallbackRef} id="camera-view"/>
+    <canvas className={className} ref={canvasCallbackRef} id="camera-view" />
   );
 };

frontend/apps/editor3d/app/src/pages/PageEnigma/comps/SceneContainer/SceneContainer.tsx

@@ -2,28 +2,22 @@ import { useCallback, useContext } from "react";
 import { useSignals } from "@preact/signals-react/runtime";
 import { pageHeight, pageWidth } from "~/signals";
 import { editorLetterBox, timelineHeight } from "~/pages/PageEnigma/signals";
-import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
+import { EngineContext, sceneContainerSignal } from "~/pages/PageEnigma/contexts/EngineContext";
 import { Letterbox } from "./Letterbox";
 
 export const SceneContainer = ({ children }: { children: React.ReactNode }) => {
   useSignals();
-  const editorEngine = useContext(EngineContext);
-
   const containerWidth = pageWidth.value;
 
   const containerHeight = pageHeight.value - timelineHeight.value - 56;
 
   const callbackRef = useCallback(
     (node: HTMLDivElement) => {
-      if (node && editorEngine) {
-        if (!editorEngine.container) {
-          editorEngine.setSceneContainer(node);
-        } else {
-          editorEngine.updateSceneContainer(node);
-        }
+      if (node) {
+        sceneContainerSignal.value = node;
       }
     },
-    [editorEngine],
+    [],
   );
 
   return (

frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineContext.tsx

@@ -13,4 +13,4 @@ export interface EditorExpandedI extends Editor {
   >;
 }
 
-export const EngineContext = createContext<EditorExpandedI | null>(null);
+export const EngineContext = createContext<Editor | null>(null);

frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx

@@ -1,56 +1,60 @@
-import { ReactNode, useEffect, useState } from "react";
+import { ReactNode, useEffect, useRef, useState } from "react";
 import { EngineContext, EditorExpandedI } from "./EngineContext";
 
 import Editor from "~/pages/PageEnigma/Editor/editor";
+import { useSignals } from "@preact/signals-react/runtime";
+import { appTabId } from "~/signals/appTab";
+import { signal } from "@preact/signals-react";
 
 interface Props {
   sceneToken?: string;
   children: ReactNode;
 }
 
+export const sceneContainerSignal = signal<HTMLDivElement | null>(null);
+export const editorCanvasSignal = signal<HTMLCanvasElement | null>(null);
+export const camViewCanvasSignal = signal<HTMLCanvasElement | null>(null);
+
 export const EngineProvider = ({ sceneToken, children }: Props) => {
-  const [editor, setEditor] = useState<EditorExpandedI | null>(null);
-  const [sceneContainer, setSceneContainer] = useState<HTMLDivElement | null>(
-    null,
-  );
-  const [editorCanvas, setEditorCanvas] = useState<HTMLCanvasElement | null>(
-    null,
-  );
-  const [camViewCanvas, setCamViewCanvas] = useState<HTMLCanvasElement | null>(
-    null,
-  );
+  useSignals();
+
+  const [editor, setEditor] = useState<Editor | null>(null);
+  const activeEditorRef = useRef<Editor | null>(null);
+
+  const createEditor = () => {
+    return new Editor();
+  }
+
+  const currentTabId = appTabId.value;
+  const sceneContainer = sceneContainerSignal.value;
+  const editorCanvas = editorCanvasSignal.value;
+  const camViewCanvas = camViewCanvasSignal.value;
   useEffect(() => {
-    setEditor((curr) => {
-      if (curr !== null) {
-        console.warn("Editor Engine already exist");
-        return curr;
+    if (sceneContainer && editorCanvas && camViewCanvas && currentTabId === "3D") {
+      // DO NOTHING if another useEffect already created one and hasn't been removed
+      if (activeEditorRef.current) {
+        return;
       }
 
-      class EditorExpanded extends Editor implements EditorExpandedI {
-        setSceneContainer = setSceneContainer;
-        setEditorCanvas = setEditorCanvas;
-        setCamViewCanvas = setCamViewCanvas;
-      }
-      return new EditorExpanded();
-    });
-  }, []);
-
-  useEffect(() => {
-    if (
-      editor &&
-      editor.can_initialize &&
-      editorCanvas &&
-      camViewCanvas &&
-      sceneContainer
-    ) {
-      editor.initialize({
+      const newEditor = createEditor();
+      console.warn("Creating new Editor instance", newEditor);
+      activeEditorRef.current = newEditor;
+      newEditor.initialize({
         sceneToken: sceneToken || "",
         sceneContainerEl: sceneContainer,
         editorCanvasEl: editorCanvas,
         camViewCanvasEl: camViewCanvas,
-      });
+      })
+      setEditor(newEditor);
+    } else if (currentTabId !== "3D") {
+      // If the tab is changed, we unmount the editor and clear all input params
+      activeEditorRef.current?.unmountEngine();
+      activeEditorRef.current = null;
+      sceneContainerSignal.value = null;
+      editorCanvasSignal.value = null;
+      camViewCanvasSignal.value = null;
     }
-  }, [editor, sceneToken, editorCanvas, camViewCanvas, sceneContainer]);
+  }, [sceneToken, editorCanvas, camViewCanvas, setEditor, sceneContainer, currentTabId]);
 
   return (
     <EngineContext.Provider value={editor}>{children}</EngineContext.Provider>

frontend/apps/editor3d/app/src/signals/appTab.ts

@@ -20,7 +20,11 @@ export const setAppTabId = (newId: string) => {
 };
 
 export const is3DEditorInitialized = signal(false);
-
 export const setIs3DEditorInitialized = (isInitialized: boolean) => {
   is3DEditorInitialized.value = isInitialized;
 };
+
+export const is3DPageMounted = signal(false);
+export const set3DPageMounted = (isMounted: boolean) => {
+  is3DPageMounted.value = isMounted;
+}

frontend/apps/editor3d/app/src/signals/loadingBar.ts

@@ -1,6 +1,6 @@
 import { signal } from "@preact/signals-core";
 
-export const loadingBarIsShowing = signal(true);
+export const loadingBarIsShowing = signal(false);
 export const loadingBarData = signal<{
   label: string;
   message: string;