Compare commits

...
Author SHA1 Message Date
Tanish Manku 72f960a5d4 Make 3D cache save cancellable with flag 2025-06-06 22:48:58 -07:00
Tanish Manku 3de468389e Make 3D state save and load on tab change 2025-06-06 17:13:31 -07:00
8 changed files with 211 additions and 118 deletions
@@ -1,4 +1,4 @@
import { useState } from "react";
import { useState } from "react";
import {
faGear,
faImages,
@@ -23,19 +23,21 @@ import {
MenuIconSelector,
MenuIconItem,
} from "@storyteller/ui-menu-icon-selector";
import { signal } from "@preact/signals-react";
import { signal } from "@preact/signals-react";
import { useSignals } from "@preact/signals-react/runtime";
import { setLogoutStates } from "~/signals/authentication/utilities";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { useTabStore } from "~/pages/Stores/TabState";
import { set3DPageMounted } from "~/pages/PageEnigma/Editor/editor";
import {
is3DEditorInitialized,
is3DSceneLoaded,
set3DPageMounted,
} from "~/pages/PageEnigma/Editor/editor";
interface Props {
pageName: string;
}
const appMenuTabs: MenuIconItem[] = [
{
id: "2D",
@@ -62,9 +64,7 @@ const appMenuTabs: MenuIconItem[] = [
export const topNavMediaId = signal<string>("");
export const topNavMediaUrl = signal<string>("");
export const TopBar = ({
pageName
}: Props) => {
export const TopBar = ({ pageName }: Props) => {
useSignals();
const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
@@ -73,9 +73,19 @@ export const TopBar = ({
galleryModalVisibleViewMode.value = true;
galleryModalVisibleDuringDrag.value = true;
};
const tabStore = useTabStore();
const is3DSceneReady = is3DSceneLoaded.value;
const is3DEditorReady = is3DEditorInitialized.value;
const disableTabSwitcher = () => {
return (
useTabStore.getState().activeTabId === "3D" &&
!is3DEditorReady &&
!is3DSceneReady
);
};
return (
<>
<header className="fixed left-0 top-0 z-[60] w-full border-b border-white/5 bg-ui-background">
@@ -95,7 +105,7 @@ export const TopBar = ({
<MenuIconSelector
menuItems={appMenuTabs}
activeMenu={tabStore.activeTabId}
disabled={false}
disabled={disableTabSwitcher()}
onMenuChange={(tabId) => {
// Disable 3d engine to prevent memory leak.
if (tabId === "3D") {
@@ -103,9 +113,7 @@ export const TopBar = ({
} else {
set3DPageMounted(false);
}
useTabStore
.getState()
.setActiveTab(tabId);
useTabStore.getState().setActiveTab(tabId);
}}
className="w-fit"
/>
@@ -158,10 +166,7 @@ export const TopBar = ({
globalAccountLogoutCallback={() => setLogoutStates()}
/>
<GalleryModal
mode="view"
onDownloadClicked={downloadFileFromUrl}
/>
<GalleryModal mode="view" onDownloadClicked={downloadFileFromUrl} />
</>
);
};
@@ -0,0 +1,32 @@
import { getArtStyle } from "~/enums/ArtStyle";
import {
cameraAspectRatio,
cinematic,
enginePreProcessing,
faceDetail,
globalIPAMediaToken,
lipSync,
styleStrength,
upscale,
} from "~/pages/PageEnigma/signals";
import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
import Editor from "./editor";
export const getSceneGenerationMetaData = (
editorEngine: Editor,
): SceneGenerationMetaData => {
// when this is called, editor engine is guarunteed by it's caller
return {
positivePrompt: editorEngine.positive_prompt,
negativePrompt: editorEngine.negative_prompt,
artisticStyle: getArtStyle(editorEngine.art_style.toString()),
cameraAspectRatio: cameraAspectRatio.value,
globalIPAMediaToken: globalIPAMediaToken.value || undefined,
upscale: upscale.value,
faceDetail: faceDetail.value,
styleStrength: styleStrength.value,
lipSync: lipSync.value,
cinematic: cinematic.value,
enginePreProcessing: enginePreProcessing.value,
};
};
@@ -50,29 +50,37 @@ import { SceneManager } from "./scene_manager_api";
import { CustomOutlinePass } from "./CustomOutlinePass.js";
import FindSurfaces from "./FindSurfaces.js";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
export type EditorInitializeConfig = {
sceneToken: string;
editorCanvasEl: HTMLCanvasElement;
camViewCanvasEl: HTMLCanvasElement;
sceneContainerEl: HTMLDivElement;
cacheJsonString?: string;
};
// ONLY USED IN THIS CLASS
// ONLY USED IN THIS CLASS
// Need this one to ensure the canvas is inflated and non-zero sized
const is3DPageMounted = signal(false);
export const set3DPageMounted = (isMounted: boolean) => {
is3DPageMounted.value = isMounted;
}
};
// Need this one to know when the engine has initialized (but not necessarily loaded saves)
export const is3DEditorInitialized = signal(false);
export const setIs3DEditorInitialized = (isInitialized: boolean) => {
is3DEditorInitialized.value = isInitialized;
};
// Need this one to know when the engine is done loading stuff into the scene for display
export const is3DSceneLoaded = signal(false);
export const setIs3DSceneLoaded = (isLoaded: boolean) => {
is3DSceneLoaded.value = isLoaded;
};
class Editor {
version: number;
activeScene: Scene;
@@ -333,9 +341,7 @@ class Editor {
// set image type at this stage
this.renderIndex = 0;
}
// Add helper method to convert focal length to FOV
@@ -400,7 +406,7 @@ class Editor {
}
updateSceneContainer(newContainer: HTMLDivElement) {
this.container = newContainer;
this.containerResizeObserver?.disconnect()
this.containerResizeObserver?.disconnect();
this.containerResizeObserver?.observe(this.container);
}
engineCanvasMayReset() {
@@ -462,6 +468,7 @@ class Editor {
editorCanvasEl,
camViewCanvasEl,
sceneContainerEl,
cacheJsonString: cacheJson = "",
}: EditorInitializeConfig) {
if (!this.can_initialize) {
console.log("Reinitializing 3D editor");
@@ -662,8 +669,15 @@ class Editor {
// Attach freecam state controller
this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
if (!this.utils.isEmpty(sceneToken)) {
this.loadScene(sceneToken);
const onloadCallback = () => {
console.log("Setting Scene is loaded");
setIs3DSceneLoaded(true);
};
if (!this.utils.isEmpty(cacheJson)) {
this.loadCache(cacheJson).then(onloadCallback);
} else if (!this.utils.isEmpty(sceneToken)) {
this.loadScene(sceneToken).then(onloadCallback);
} else {
signalScene({
title: "Untitled New Scene",
@@ -671,6 +685,7 @@ class Editor {
ownerToken: authentication.userInfo.value?.user_token,
isModified: false,
});
onloadCallback();
}
this._configurePostProcessingRaw();
@@ -917,6 +932,10 @@ class Editor {
}
}
public async loadCache(cacheJson: string) {
await this.save_manager.loadCache(cacheJson);
}
public async loadScene(scene_media_token: string) {
await this.save_manager.loadScene(scene_media_token);
@@ -981,6 +1000,20 @@ class Editor {
});
}
public cacheScene({
sceneTitle,
sceneToken,
sceneGenerationMetadata,
}: {
sceneTitle: string;
sceneToken: string;
sceneGenerationMetadata: SceneGenereationMetaData;
}) {
return this.save_manager.getSceneJson({
sceneGenerationMetadata: sceneGenerationMetadata,
});
}
get_camera_person_mode(): boolean {
return this.camera_person_mode;
}
@@ -1183,10 +1216,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 576
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
@@ -1195,10 +1228,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 1024
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 900
: 1000;
@@ -1261,7 +1294,6 @@ class Editor {
console.log(`COLLECTING COLOR FRAMES COUNTED: ${this.renderIndex}`);
await this.engineFrameBuffers.collectColorFrames(this.renderIndex);
this.renderIndex = 0;
console.time("Stop Playback And Upload Video Time");
await this.stopPlaybackAndUploadVideo();
@@ -1467,11 +1499,20 @@ class Editor {
}
unmountEngine() {
setIs3DSceneLoaded(false);
this.stopRenderLoop();
this.sceneManager?.detachEventListeners();
this.disableFreeCamControls();
this.cameraViewControls?.detachEventListeners();
// Fix: dispose 3D contexts
this.renderer?.dispose();
this.composer?.dispose();
this.render_composer?.dispose();
this.rawRenderer?.dispose();
this.isMounted = false;
setIs3DEditorInitialized(false);
console.log("3D Editor Engine unmounted");
}
@@ -53,8 +53,6 @@ export class SaveManager {
);
const scene_json = await proxyScene.saveToScene(this.editor.version);
const check_sum_data = {
scene: scene_json,
timeline: "",
@@ -81,8 +79,6 @@ export class SaveManager {
);
const scene_json = await proxyScene.saveToScene(this.editor.version);
// Save all cameras data
const camerasData = cameras.value.map((cam: Camera) => ({
id: cam.id,
@@ -112,28 +108,18 @@ export class SaveManager {
return jsonString;
}
// TODO Move this function into scene manager.
public async saveScene({
sceneTitle,
sceneToken,
public getSceneJson({
sceneGenerationMetadata,
}: {
sceneTitle: string;
sceneToken?: string;
sceneGenerationMetadata: SceneGenereationMetaData;
}): Promise<string> {
this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out.
// remove controls when saving scene.
this.editor.utils.removeTransformControls();
showEditorLoader();
}) {
const proxyScene = new StoryTellerProxyScene(
this.editor.version,
this.editor.activeScene,
);
const scene_json = await proxyScene.saveToScene(this.editor.version);
const scene_json = proxyScene.saveToScene(this.editor.version);
console.log(scene_json);
// Save all cameras data
const camerasData = cameras.value.map((cam: Camera) => ({
id: cam.id,
@@ -159,6 +145,28 @@ export class SaveManager {
selectedCameraId: selectedCameraId.value,
};
return save_data;
}
// TODO Move this function into scene manager.
public async saveScene({
sceneTitle,
sceneToken,
sceneGenerationMetadata,
}: {
sceneTitle: string;
sceneToken?: string;
sceneGenerationMetadata: SceneGenereationMetaData;
}): Promise<string> {
this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out.
// remove controls when saving scene.
this.editor.utils.removeTransformControls();
showEditorLoader();
const sceneJson = this.getSceneJson({
sceneGenerationMetadata,
});
// TODO turn scene information into and object ...
let sceneThumbnail = undefined;
@@ -169,7 +177,7 @@ export class SaveManager {
}
const result = await this.editor.api_manager.saveSceneState({
saveJson: JSON.stringify(save_data),
saveJson: JSON.stringify(sceneJson),
sceneTitle,
sceneToken,
sceneThumbnail,
@@ -186,19 +194,7 @@ export class SaveManager {
return result; // if this is an empty string it is an error. need to migrate to api manager.
}
// TODO Refactor remove editor.
public async loadScene(scene_media_token: string) {
showEditorLoader();
this.editor.current_scene_media_token = scene_media_token;
const scene_json = await this.editor.api_manager
.loadSceneState(this.editor.current_scene_media_token)
.catch((err) => {
hideEditorLoader();
throw err;
});
private async loadFromJson(scene_json: any) {
const proxyScene = new StoryTellerProxyScene(
this.editor.version,
this.editor.activeScene,
@@ -273,6 +269,32 @@ export class SaveManager {
console.debug("Loading Timeline", scene_json["timeline"]);
this.editor.timeline.checkEditorCanPlay();
}
public async loadCache(cacheJson: string) {
showEditorLoader();
const scene_json = JSON.parse(cacheJson);
await this.loadFromJson(scene_json);
hideEditorLoader();
this.editor.timeline.scrub({ currentTime: 0 });
}
// TODO Refactor remove editor.
public async loadScene(scene_media_token: string) {
showEditorLoader();
this.editor.current_scene_media_token = scene_media_token;
const scene_json = await this.editor.api_manager
.loadSceneState(this.editor.current_scene_media_token)
.catch((err) => {
hideEditorLoader();
throw err;
});
await this.loadFromJson(scene_json);
hideEditorLoader();
// TODO figure out if this is a bug.
@@ -7,11 +7,8 @@ import {
faQuestion,
faSquare,
} from "@fortawesome/pro-solid-svg-icons";
import {
EditorExpandedI,
EngineContext,
} from "~/pages/PageEnigma/contexts/EngineContext";
import { ToastTypes, getArtStyle } from "~/enums";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
import { ToastTypes } from "~/enums";
import { scene, signalScene, authentication, addToast } from "~/signals";
import { outlinerIsShowing } from "~/pages/PageEnigma/signals/outliner/outliner";
import { ButtonDropdown } from "@storyteller/ui-button-dropdown";
@@ -20,22 +17,12 @@ import { Button } from "@storyteller/ui-button";
import { TestFeaturesButtons } from "./TestFeaturesButtons";
import { LoadUserScenes } from "./LoadUserScenes";
import { getCurrentLocationWithoutParams, isNumberString } from "~/utilities";
import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
import {
cameraAspectRatio,
cinematic,
enginePreProcessing,
faceDetail,
globalIPAMediaToken,
lipSync,
resetSceneGenerationMetadata,
styleStrength,
upscale,
} from "~/pages/PageEnigma/signals";
import { resetSceneGenerationMetadata } from "~/pages/PageEnigma/signals";
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
import { twMerge } from "tailwind-merge";
import { Help } from "./Help";
import { Modal } from "@storyteller/ui-modal";
import { getSceneGenerationMetaData } from "../../Editor/SceneMetadata";
export const ControlsTopButtons = () => {
useSignals();
@@ -61,32 +48,12 @@ export const ControlsTopButtons = () => {
);
};
const getSceneGenereationMetaData = useCallback(
(editorEngine: EditorExpandedI): SceneGenerationMetaData => {
// when this is called, editor engine is guarunteed by it's caller
return {
positivePrompt: editorEngine.positive_prompt,
negativePrompt: editorEngine.negative_prompt,
artisticStyle: getArtStyle(editorEngine.art_style.toString()),
cameraAspectRatio: cameraAspectRatio.value,
globalIPAMediaToken: globalIPAMediaToken.value || undefined,
upscale: upscale.value,
faceDetail: faceDetail.value,
styleStrength: styleStrength.value,
lipSync: lipSync.value,
cinematic: cinematic.value,
enginePreProcessing: enginePreProcessing.value,
};
},
[],
);
const handleButtonSave = async () => {
if (!editorEngine) {
addToast(ToastTypes.ERROR, "No Engine Error in Saving Scenes");
return;
}
const sceneGenerationMetadata = getSceneGenereationMetaData(editorEngine);
const sceneGenerationMetadata = getSceneGenerationMetaData(editorEngine);
const retSceneMediaToken = await editorEngine.saveScene({
sceneTitle: scene.value.title || "",
@@ -114,7 +81,7 @@ export const ControlsTopButtons = () => {
addToast(ToastTypes.ERROR, "No Engine Error in Saving Scenes");
return;
}
const sceneGenerationMetadata = getSceneGenereationMetaData(editorEngine);
const sceneGenerationMetadata = getSceneGenerationMetaData(editorEngine);
const retSceneMediaToken = await editorEngine.saveScene({
sceneTitle: sceneTitleInput,
sceneToken: undefined,
@@ -129,7 +96,7 @@ export const ControlsTopButtons = () => {
title: sceneTitleInput,
});
}
}, [sceneTitleInput, editorEngine, getSceneGenereationMetaData]);
}, [sceneTitleInput, editorEngine, getSceneGenerationMetaData]);
const handleButtonLoadScene = () => {
handleResetScene();
@@ -1,11 +1,12 @@
import { ReactNode, useEffect, useRef, useState } from "react";
import { EngineContext } from "./EngineContext";
import Editor from "~/pages/PageEnigma/Editor/editor";
import Editor, { is3DSceneLoaded } from "~/pages/PageEnigma/Editor/editor";
import { useSignals } from "@preact/signals-react/runtime";
import { signal } from "@preact/signals-react";
import { useTabStore } from "../../../Stores/TabState"
import { useTabStore } from "../../../Stores/TabState";
import { getSceneGenerationMetaData } from "../../Editor/SceneMetadata";
interface Props {
sceneToken?: string;
children: ReactNode;
@@ -23,10 +24,10 @@ export const EngineProvider = ({ sceneToken, children }: Props) => {
const createEditor = () => {
return new Editor();
}
};
const tabStore = useTabStore();
const tab = tabStore.activeTabId;
const sceneContainer = sceneContainerSignal.value;
const editorCanvas = editorCanvasSignal.value;
const camViewCanvas = camViewCanvasSignal.value;
@@ -36,19 +37,44 @@ export const EngineProvider = ({ sceneToken, children }: Props) => {
if (activeEditorRef.current) {
return;
}
const newEditor = createEditor();
console.warn("Creating new Editor instance", newEditor);
activeEditorRef.current = newEditor;
// Check if we have a cached state
const cacheString = tabStore.getTabData("3D") as string;
newEditor.initialize({
sceneToken: sceneToken || "",
sceneContainerEl: sceneContainer,
editorCanvasEl: editorCanvas,
camViewCanvasEl: camViewCanvas,
})
cacheJsonString: cacheString,
});
setEditor(newEditor);
} else if (tab !== "3D") {
// If the tab is changed, we unmount the editor and clear all input params
if (!activeEditorRef.current) {
return;
}
// If the tab is changed, cache state, unmount the editor and clear all input params
// This condition makes sure the editor saves a fully loaded scene so it doesn't lose data
// If the load was incomplete, we'll preserve the last cache anyway.
if (is3DSceneLoaded.peek()) {
const sceneGenerationMetadata = getSceneGenerationMetaData(
activeEditorRef.current,
);
const cacheJson = activeEditorRef.current.cacheScene({
sceneTitle: "",
sceneToken: "",
sceneGenerationMetadata: sceneGenerationMetadata,
});
const cacheString = JSON.stringify(cacheJson);
tabStore.updateTabData("3D", cacheString);
}
// Unmount/Destructor flow
activeEditorRef.current?.unmountEngine();
activeEditorRef.current = null;
sceneContainerSignal.value = null;
@@ -105,7 +105,7 @@ export class StoryTellerProxy3DObject {
this.locked = object.userData["locked"];
}
public async toJSON(): Promise<ObjectJSON> {
public toJSON(): ObjectJSON {
const json: ObjectJSON = {
version: this.version,
position: {
@@ -27,7 +27,7 @@ export class StoryTellerProxyScene {
this.sceneItemProxy = [];
}
async getChildren(child: THREE.Object3D) {
getChildren(child: THREE.Object3D) {
if (this.lookUpDictionary[child.uuid] == null) {
this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject(
this.version,
@@ -54,11 +54,11 @@ export class StoryTellerProxyScene {
const boneHelper = new BoneJSONHelper(child);
proxyObject3D.rigData = boneHelper.toJSON();
const json_data = await proxyObject3D.toJSON();
const json_data = proxyObject3D.toJSON();
return json_data;
}
public async saveToSceneOlder(): Promise<any> {
public saveToSceneOlder(): ObjectJSON[] {
const results: ObjectJSON[] = [];
if (this.scene.scene != null) {
for (const child of this.scene.scene.children) {
@@ -82,7 +82,7 @@ export class StoryTellerProxyScene {
proxyObject3D.shininess = child.userData["shininess"];
proxyObject3D.specular = child.userData["specular"];
proxyObject3D.locked = child.userData["locked"];
const json_data = await proxyObject3D.toJSON();
const json_data = proxyObject3D.toJSON();
results.push(json_data);
}
}
@@ -93,7 +93,7 @@ export class StoryTellerProxyScene {
return results;
}
public async saveToScene(version: number): Promise<any> {
public saveToScene(version: number): ObjectJSON[] {
this.version = version;
console.log("Saving with version:", this.version);
const results: ObjectJSON[] = [];
@@ -101,8 +101,8 @@ export class StoryTellerProxyScene {
for (const pchild of this.scene.scene.children) {
if (this.version >= 1.0) {
if (pchild.userData["media_id"] != undefined) {
console.debug("Object JSON:", pchild.toJSON())
results.push(await this.getChildren(pchild));
console.debug("Object JSON:", pchild.toJSON());
results.push(this.getChildren(pchild));
}
} else {
console.log("Saving older.");