diff --git a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts index b38348ea84..ca1dfaf42e 100644 --- a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts +++ b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts @@ -2,145 +2,7 @@ import { create } from "zustand"; import { listen } from "@tauri-apps/api/event"; import { MediaFilesApi, MediaUploadApi } from "@storyteller/api"; import { v4 as uuidv4 } from "uuid"; -import * as THREE from "three"; -import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; - -// Offscreen thumbnail capture function - produces a square image -async function captureModelThumbnail(modelUrl: string): Promise { - return new Promise((resolve) => { - try { - // Square dimensions for thumbnail - const size = 512; - const fov = 50; - - const scene = new THREE.Scene(); - scene.background = new THREE.Color(0x282828); - - const camera = new THREE.PerspectiveCamera(fov, 1, 0.1, 1000); // Aspect ratio 1 for square - - const renderer = new THREE.WebGLRenderer({ - antialias: true, - alpha: false, - preserveDrawingBuffer: true, - }); - renderer.setSize(size, size); - renderer.setPixelRatio(1); - - // Add lights - const ambientLight = new THREE.AmbientLight(0xffffff, 2); - scene.add(ambientLight); - - const hemisphereLight = new THREE.HemisphereLight( - 0xffffff, - 0x888888, - 1.2, - ); - scene.add(hemisphereLight); - - const keyLight = new THREE.DirectionalLight(0xffffff, 2); - keyLight.position.set(2, 10, 8); - scene.add(keyLight); - - const fillLight = new THREE.DirectionalLight(0xffffff, 1.2); - fillLight.position.set(-6, 6, -4); - scene.add(fillLight); - - const frontLight = new THREE.DirectionalLight(0xffffff, 1); - frontLight.position.set(0, 4, 10); - scene.add(frontLight); - - const loader = new GLTFLoader(); - // Without anonymous CORS the GLTF + textures taint the WebGL canvas, so - // toDataURL() below throws a SecurityError and the thumbnail is lost. - loader.setCrossOrigin("anonymous"); - loader.load( - modelUrl, - (gltf) => { - const model = gltf.scene; - - const box = new THREE.Box3().setFromObject(model); - const modelSize = box.getSize(new THREE.Vector3()); - - const maxDim = Math.max(modelSize.x, modelSize.y, modelSize.z); - const scale = 2 / maxDim; - model.scale.multiplyScalar(scale); - - const scaledBox = new THREE.Box3().setFromObject(model); - const scaledCenter = scaledBox.getCenter(new THREE.Vector3()); - const scaledSize = scaledBox.getSize(new THREE.Vector3()); - - model.position.x = -scaledCenter.x; - model.position.z = -scaledCenter.z; - model.position.y = -scaledBox.min.y; - - scene.add(model); - - // Calculate camera distance to fit model in view with padding - const modelHeight = scaledSize.y; - const maxModelDim = Math.max( - scaledSize.x, - scaledSize.y, - scaledSize.z, - ); - - // Calculate distance needed to fit the model based on FOV - // For a 45-degree camera angle, we view from a diagonal - const fovRad = (fov * Math.PI) / 180; - const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2); - - // Add some padding (1.3x) and account for diagonal viewing angle - const cameraDistance = fitDistance * 1.3; - - // Position camera at 45-degree angle - const angle = Math.PI / 4; // 45 degrees - camera.position.set( - Math.sin(angle) * cameraDistance, - modelHeight * 0.5 + cameraDistance * 0.4, - Math.cos(angle) * cameraDistance, - ); - camera.lookAt(0, modelHeight * 0.4, 0); - - // Render and capture - renderer.render(scene, camera); - let dataUrl: string | null = null; - try { - dataUrl = renderer.domElement.toDataURL("image/png"); - } catch (err) { - // A tainted canvas (CORS) throws SecurityError here. Log so the - // failure is diagnosable instead of silently producing no thumbnail. - console.warn( - "[captureModelThumbnail] toDataURL failed (likely CORS taint):", - err, - ); - } - - // Cleanup - renderer.dispose(); - scene.clear(); - - resolve(dataUrl); - }, - undefined, - (error) => { - console.error("[captureModelThumbnail] Error loading model:", error); - renderer.dispose(); - scene.clear(); - resolve(null); - }, - ); - - // Timeout after 30 seconds - setTimeout(() => { - renderer.dispose(); - scene.clear(); - resolve(null); - }, 30000); - } catch (error) { - console.error("[captureModelThumbnail] Error:", error); - resolve(null); - } - }); -} +import { captureModelThumbnail } from "./captureModelThumbnail"; export interface PendingExternalImage { url: string; diff --git a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.spec.ts b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.spec.ts new file mode 100644 index 0000000000..b6345a6884 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.spec.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as THREE from "three"; +import { captureModelThumbnail } from "./captureModelThumbnail"; + +const { renderers, pendingLoads } = vi.hoisted(() => ({ + renderers: [] as Array<{ + dispose: ReturnType; + forceContextLoss: ReturnType; + }>, + pendingLoads: [] as Array<{ + onLoad: (gltf: { scene: THREE.Object3D }) => void; + onError: (error: unknown) => void; + }>, +})); + +// jsdom has no WebGL; only the teardown calls matter here. +vi.mock("three", async (importOriginal) => { + const actual = await importOriginal(); + class WebGLRenderer { + domElement = { toDataURL: () => "data:image/png;base64,AAAA" }; + dispose = vi.fn(); + forceContextLoss = vi.fn(); + constructor() { + renderers.push(this); + } + setSize() {} + setPixelRatio() {} + render() {} + } + return { ...actual, WebGLRenderer }; +}); + +// Loads stay pending until a test settles them. +vi.mock("three/examples/jsm/loaders/GLTFLoader.js", () => ({ + GLTFLoader: class { + setCrossOrigin() {} + load( + _url: string, + onLoad: (gltf: { scene: THREE.Object3D }) => void, + _onProgress: unknown, + onError: (error: unknown) => void, + ) { + pendingLoads.push({ onLoad, onError }); + } + }, +})); + +describe("captureModelThumbnail", () => { + beforeEach(() => { + renderers.length = 0; + pendingLoads.length = 0; + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("returns the frame and frees the model, renderer and context once", async () => { + const { model, onDispose } = trackedModel(); + const capture = captureModelThumbnail("model.glb"); + + pendingLoads[0].onLoad({ scene: model }); + const dataUrl = await capture; + vi.runAllTimers(); + + expect(dataUrl).toMatch(/^data:image\/png/); + expect(onDispose).toHaveBeenCalledTimes(1); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); + + it("frees the renderer and context when the model fails to load", async () => { + const capture = captureModelThumbnail("missing.glb"); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + pendingLoads[0].onError(new Error("404")); + + expect(await capture).toBeNull(); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); + + it("gives up after the timeout and frees a model that loads later", async () => { + const capture = captureModelThumbnail("slow.glb"); + + vi.advanceTimersByTime(30000); + expect(await capture).toBeNull(); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + + const { model, onDispose } = trackedModel(); + pendingLoads[0].onLoad({ scene: model }); + + expect(onDispose).toHaveBeenCalledTimes(1); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + }); +}); + +function trackedModel() { + const geometry = new THREE.BoxGeometry(); + const onDispose = vi.fn(); + geometry.addEventListener("dispose", onDispose); + const model = new THREE.Group(); + model.add(new THREE.Mesh(geometry)); + return { model, onDispose }; +} diff --git a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.ts b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.ts new file mode 100644 index 0000000000..d67d863d19 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.ts @@ -0,0 +1,160 @@ +import * as THREE from "three"; +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; +import { disposeObject3D } from "@storyteller/ui-viewer-3d"; + +const THUMBNAIL_SIZE = 512; +const FOV = 50; +const LOAD_TIMEOUT_MS = 30000; + +// Offscreen thumbnail capture: loads the model into a throwaway renderer, +// renders one square frame and returns it as a PNG data URL (null on +// failure or timeout). The renderer, its WebGL context and the model are +// freed exactly once, whichever of load, error or timeout comes first. +export async function captureModelThumbnail( + modelUrl: string, +): Promise { + return new Promise((resolve) => { + try { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x282828); + + const camera = new THREE.PerspectiveCamera(FOV, 1, 0.1, 1000); // Aspect ratio 1 for square + + const renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: false, + preserveDrawingBuffer: true, + }); + renderer.setSize(THUMBNAIL_SIZE, THUMBNAIL_SIZE); + renderer.setPixelRatio(1); + + addLights(scene); + + let done = false; + const finish = (dataUrl: string | null) => { + if (done) return; + done = true; + clearTimeout(timeout); + disposeObject3D(scene); + renderer.dispose(); + // The renderer's canvas is never attached, so nothing else can use + // this context; release it now instead of at garbage collection. + renderer.forceContextLoss(); + resolve(dataUrl); + }; + + const timeout = setTimeout(() => finish(null), LOAD_TIMEOUT_MS); + + const loader = new GLTFLoader(); + // Without anonymous CORS the GLTF + textures taint the WebGL canvas, so + // toDataURL() below throws a SecurityError and the thumbnail is lost. + loader.setCrossOrigin("anonymous"); + loader.load( + modelUrl, + (gltf) => { + // Timed out already: the renderer is gone, just free the model. + if (done) { + disposeObject3D(gltf.scene); + return; + } + const model = gltf.scene; + const modelHeight = fitModel(model); + scene.add(model); + frameCamera(camera, model, modelHeight); + + // Render and capture + renderer.render(scene, camera); + let dataUrl: string | null = null; + try { + dataUrl = renderer.domElement.toDataURL("image/png"); + } catch (err) { + // A tainted canvas (CORS) throws SecurityError here. Log so the + // failure is diagnosable instead of silently producing no thumbnail. + console.warn( + "[captureModelThumbnail] toDataURL failed (likely CORS taint):", + err, + ); + } + finish(dataUrl); + }, + undefined, + (error) => { + console.error("[captureModelThumbnail] Error loading model:", error); + finish(null); + }, + ); + } catch (error) { + console.error("[captureModelThumbnail] Error:", error); + resolve(null); + } + }); +} + +// Scale the model to a 2-unit max dimension and stand it centered on the +// origin. Returns its scaled height. +function fitModel(model: THREE.Object3D): number { + const box = new THREE.Box3().setFromObject(model); + const modelSize = box.getSize(new THREE.Vector3()); + + const maxDim = Math.max(modelSize.x, modelSize.y, modelSize.z); + const scale = 2 / maxDim; + model.scale.multiplyScalar(scale); + + const scaledBox = new THREE.Box3().setFromObject(model); + const scaledCenter = scaledBox.getCenter(new THREE.Vector3()); + + model.position.x = -scaledCenter.x; + model.position.z = -scaledCenter.z; + model.position.y = -scaledBox.min.y; + + return scaledBox.getSize(new THREE.Vector3()).y; +} + +// Place the camera on a 45-degree diagonal, far enough to fit the model. +function frameCamera( + camera: THREE.PerspectiveCamera, + model: THREE.Object3D, + modelHeight: number, +): void { + const scaledSize = new THREE.Box3() + .setFromObject(model) + .getSize(new THREE.Vector3()); + const maxModelDim = Math.max(scaledSize.x, scaledSize.y, scaledSize.z); + + // Calculate distance needed to fit the model based on FOV + // For a 45-degree camera angle, we view from a diagonal + const fovRad = (FOV * Math.PI) / 180; + const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2); + + // Add some padding (1.3x) and account for diagonal viewing angle + const cameraDistance = fitDistance * 1.3; + + // Position camera at 45-degree angle + const angle = Math.PI / 4; // 45 degrees + camera.position.set( + Math.sin(angle) * cameraDistance, + modelHeight * 0.5 + cameraDistance * 0.4, + Math.cos(angle) * cameraDistance, + ); + camera.lookAt(0, modelHeight * 0.4, 0); +} + +function addLights(scene: THREE.Scene): void { + const ambientLight = new THREE.AmbientLight(0xffffff, 2); + scene.add(ambientLight); + + const hemisphereLight = new THREE.HemisphereLight(0xffffff, 0x888888, 1.2); + scene.add(hemisphereLight); + + const keyLight = new THREE.DirectionalLight(0xffffff, 2); + keyLight.position.set(2, 10, 8); + scene.add(keyLight); + + const fillLight = new THREE.DirectionalLight(0xffffff, 1.2); + fillLight.position.set(-6, 6, -4); + scene.add(fillLight); + + const frontLight = new THREE.DirectionalLight(0xffffff, 1); + frontLight.position.set(0, 4, 10); + scene.add(frontLight); +} diff --git a/frontend/libs/components/pagedraw/src/lib/components/Model3DOverlay.tsx b/frontend/libs/components/pagedraw/src/lib/components/Model3DOverlay.tsx index d0a1b29278..12b5f28ed3 100644 --- a/frontend/libs/components/pagedraw/src/lib/components/Model3DOverlay.tsx +++ b/frontend/libs/components/pagedraw/src/lib/components/Model3DOverlay.tsx @@ -3,6 +3,7 @@ import * as THREE from "three"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js"; import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; import Konva from "konva"; +import { disposeObject3D } from "@storyteller/ui-viewer-3d"; import { Node } from "../Node"; import { Model3DParams, @@ -226,11 +227,17 @@ export const Model3DOverlay = React.forwardRef { + if (cancelled) { + disposeObject3D(gltf.scene); + return; + } const model = gltf.scene; // Auto-fit + apply user scale @@ -261,18 +268,15 @@ export const Model3DOverlay = React.forwardRef { + cancelled = true; if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current); controls.dispose(); - scene.traverse((obj) => { - if (obj instanceof THREE.Mesh) { - obj.geometry?.dispose(); - const mats = Array.isArray(obj.material) - ? obj.material - : [obj.material]; - mats.forEach((m) => m?.dispose()); - } - }); + disposeObject3D(scene); renderer.dispose(); + // A model change re-runs this effect on the same canvas, which hands + // the next renderer this same context, so only force the loss once + // the canvas has left the DOM (the overlay closed). + if (!canvas.isConnected) renderer.forceContextLoss(); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [node.modelUrl]); diff --git a/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.spec.ts b/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.spec.ts new file mode 100644 index 0000000000..a2d6141fd8 --- /dev/null +++ b/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.spec.ts @@ -0,0 +1,82 @@ +import * as THREE from "three"; +import { + DEFAULT_MODEL3D_PARAMS, + render3DModelToDataUrl, +} from "./render3DModel"; + +const { renderers, gltfOutcome } = vi.hoisted(() => ({ + renderers: [] as FakeRenderer[], + gltfOutcome: { scene: undefined as unknown, error: undefined as unknown }, +})); + +type FakeRenderer = { + dispose: ReturnType; + forceContextLoss: ReturnType; +}; + +// jsdom has no WebGL; only the teardown calls matter here. +vi.mock("three", async (importOriginal) => { + const actual = await importOriginal(); + class WebGLRenderer { + domElement = { toDataURL: () => "data:image/png;base64,AAAA" }; + dispose = vi.fn(); + forceContextLoss = vi.fn(); + constructor() { + renderers.push(this); + } + setClearColor() {} + setSize() {} + render() {} + } + return { ...actual, WebGLRenderer }; +}); + +vi.mock("three/examples/jsm/loaders/GLTFLoader.js", () => ({ + GLTFLoader: class { + load( + _url: string, + onLoad: (gltf: { scene: unknown }) => void, + _onProgress: unknown, + onError: (error: unknown) => void, + ) { + if (gltfOutcome.error) onError(gltfOutcome.error); + else onLoad({ scene: gltfOutcome.scene }); + } + }, +})); + +describe("render3DModelToDataUrl", () => { + beforeEach(() => { + renderers.length = 0; + gltfOutcome.scene = undefined; + gltfOutcome.error = undefined; + }); + + it("frees the model and the renderer's context after rendering", async () => { + const geometry = new THREE.BoxGeometry(); + const onDispose = vi.fn(); + geometry.addEventListener("dispose", onDispose); + gltfOutcome.scene = new THREE.Mesh(geometry); + + const dataUrl = await render3DModelToDataUrl( + "model.glb", + DEFAULT_MODEL3D_PARAMS, + ); + + expect(dataUrl).toMatch(/^data:image\/png/); + expect(onDispose).toHaveBeenCalledTimes(1); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); + + it("frees the renderer's context when the model fails to load", async () => { + gltfOutcome.error = new Error("404"); + + await expect( + render3DModelToDataUrl("missing.glb", DEFAULT_MODEL3D_PARAMS), + ).rejects.toThrow("404"); + + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.ts b/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.ts index 5906c1243e..c804b5f57c 100644 --- a/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.ts +++ b/frontend/libs/components/pagedraw/src/lib/utilities/render3DModel.ts @@ -1,5 +1,6 @@ import * as THREE from "three"; import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; +import { disposeObject3D } from "@storyteller/ui-viewer-3d"; export interface Model3DParams { cameraPosition: { x: number; y: number; z: number }; @@ -76,18 +77,16 @@ function normalizeAndPlaceModel( model.position.y = -scaledBox.min.y; } +// The renderer and its canvas are throwaway, so free the scene and then the +// context itself; otherwise each render holds a WebGL context until the +// canvas is garbage collected. function disposeScene( scene: THREE.Scene, renderer: THREE.WebGLRenderer, ): void { - scene.traverse((obj) => { - if (obj instanceof THREE.Mesh) { - obj.geometry?.dispose(); - const mats = Array.isArray(obj.material) ? obj.material : [obj.material]; - mats.forEach((m) => m?.dispose()); - } - }); + disposeObject3D(scene); renderer.dispose(); + renderer.forceContextLoss(); } export async function render3DModelToDataUrl( diff --git a/frontend/libs/components/pagedraw/tsconfig.lib.json b/frontend/libs/components/pagedraw/tsconfig.lib.json index 2d93887a85..835b56cfde 100644 --- a/frontend/libs/components/pagedraw/tsconfig.lib.json +++ b/frontend/libs/components/pagedraw/tsconfig.lib.json @@ -88,6 +88,9 @@ }, { "path": "../modal/tsconfig.lib.json" + }, + { + "path": "../viewer-3d/tsconfig.lib.json" } ] } diff --git a/frontend/libs/components/pagescene/src/lib/engine/editor.ts b/frontend/libs/components/pagescene/src/lib/engine/editor.ts index 96af58ca37..075cb39d43 100644 --- a/frontend/libs/components/pagescene/src/lib/engine/editor.ts +++ b/frontend/libs/components/pagescene/src/lib/engine/editor.ts @@ -1127,11 +1127,27 @@ class Editor { this.bus.emit(new SceneLoadedEvent(false)); this.stopRenderLoop(); + this.viewport.dispose(); + + // Free the scene's GPU resources. EngineProvider serialized the scene + // before calling us, and the next mount builds a fresh Editor, so + // nothing reads this scene again. The gizmo's dispose also removes its + // pointer listeners from the canvas. + this.entranceAnimator.clear(); + this.gizmo.control?.dispose(); + this.activeScene.dispose(); // Fix: dispose 3D contexts this.renderer?.dispose(); this.postProcessing.dispose(); this.rawRenderer?.dispose(); + // dispose() leaves the GL objects themselves (render targets, programs) + // in the context. Leaving the page detaches the canvases, and the + // context then lingers until garbage collection, two per visit. A + // scene change keeps the canvases mounted and the next Editor reuses + // their contexts, so only release a context whose canvas is gone. + releaseContextIfDetached(this.renderer); + releaseContextIfDetached(this.rawRenderer); this.isMounted = false; this.bus.emit(new EngineInitializedEvent(false)); @@ -1145,4 +1161,10 @@ class Editor { } } +function releaseContextIfDetached(renderer: THREE.WebGLRenderer | undefined) { + if (renderer && !renderer.domElement.isConnected) { + renderer.forceContextLoss(); + } +} + export default Editor; diff --git a/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.spec.ts b/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.spec.ts new file mode 100644 index 0000000000..7aca5291ad --- /dev/null +++ b/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.spec.ts @@ -0,0 +1,38 @@ +import { ViewportController } from "./ViewportController"; + +describe("ViewportController", () => { + it("stops observing the container on dispose", () => { + const observers = stubResizeObserver(); + const renderScene = vi.fn(); + const viewport = new ViewportController({ + getCamera: () => null, + getRenderCamera: () => null, + getRenderer: () => ({ setSize: vi.fn(), setPixelRatio: vi.fn() }) as never, + getRenderAspectRatio: () => 1, + resizePostProcessing: vi.fn(), + renderScene, + }); + viewport.container = document.createElement("div"); + + viewport.setupResizeObserver(); + viewport.dispose(); + + expect(observers[0].disconnect).toHaveBeenCalledTimes(1); + }); +}); + +// jsdom has no ResizeObserver; record each one the controller creates. +function stubResizeObserver() { + const observers: Array<{ disconnect: ReturnType }> = []; + vi.stubGlobal( + "ResizeObserver", + class { + disconnect = vi.fn(); + constructor() { + observers.push(this); + } + observe() {} + }, + ); + return observers; +} diff --git a/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.ts b/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.ts index e23a3184a2..a2ff8280a4 100644 --- a/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.ts +++ b/frontend/libs/components/pagescene/src/lib/engine/editor/ViewportController.ts @@ -100,4 +100,12 @@ export class ViewportController { }); this.observer.observe(this.container); } + + // Stop observing on unmount. The container can outlive this editor (a + // scene change keeps it mounted), and a live observer would keep + // re-rendering the dead editor's scene on every resize. + dispose() { + this.observer?.disconnect(); + this.observer = undefined; + } } diff --git a/frontend/libs/components/pagescene/src/lib/engine/helper.ts b/frontend/libs/components/pagescene/src/lib/engine/helper.ts index b90ef1d826..8df19a90fa 100644 --- a/frontend/libs/components/pagescene/src/lib/engine/helper.ts +++ b/frontend/libs/components/pagescene/src/lib/engine/helper.ts @@ -148,45 +148,11 @@ function removeObject3D(object3D) { return; } - // Finally remove the object from the scene + // Finally remove the object from the scene and free its GPU resources. + // Undo re-loads deleted objects from a snapshot (DeleteAction), so + // nothing reuses this instance. this.scene.scene.remove(obj); - - obj.traverse(child => { - (child as THREE.Mesh)?.geometry?.dispose() - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const childTexture = (child as any).texture; - if (Array.isArray(childTexture)) { - childTexture.forEach(mat => mat.dispose()); - } else if (childTexture) { - childTexture.dispose(); - } - - const childMaterial = (child as THREE.Mesh).material; - if (Array.isArray(childMaterial)) { - childMaterial.forEach(mat => mat.dispose()); - } else if (childMaterial) { - childMaterial.dispose(); - } - }) - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const objTexture = (obj as any).texture; - if (Array.isArray(objTexture)) { - objTexture.forEach(mat => mat.dispose()); - } else if (objTexture) { - objTexture.dispose(); - } - - const objMaterial = (obj as THREE.Mesh).material; - if (Array.isArray(objMaterial)) { - objMaterial.forEach(mat => mat.dispose()); - } else if (objMaterial) { - objMaterial.dispose(); - } - - if ((obj as THREE.Mesh).geometry) { - (obj as THREE.Mesh).geometry.dispose() - } + this.scene.disposeObject(obj); this.deps.bus.emit(new ObjectRemovedEvent(uuid)); this.deps.clearSelected(); diff --git a/frontend/libs/components/pagescene/src/lib/engine/scene.spec.ts b/frontend/libs/components/pagescene/src/lib/engine/scene.spec.ts new file mode 100644 index 0000000000..37b9731f6b --- /dev/null +++ b/frontend/libs/components/pagescene/src/lib/engine/scene.spec.ts @@ -0,0 +1,167 @@ +import * as THREE from "three"; +import Scene from "./scene"; + +// Scene imports Spark for splat loading; none of these tests load one. +vi.mock("@sparkjsdev/spark", async () => { + const { Object3D } = await import("three"); + return { + SplatMesh: class extends Object3D {}, + SparkRenderer: class extends Object3D {}, + }; +}); + +// The Addons barrel pulls in a lottie canvas module jsdom cannot run; Scene +// only needs these two names from it. +vi.mock("three/examples/jsm/Addons.js", async () => { + const { Mesh } = await import("three"); + return { MMDAnimationHelper: class {}, Water: class extends Mesh {} }; +}); + +describe("Scene disposal", () => { + describe("disposeObject", () => { + it("frees the object's GPU resources", () => { + const scene = makeScene(); + const geometry = new THREE.BoxGeometry(); + const onDispose = vi.fn(); + geometry.addEventListener("dispose", onDispose); + + scene.disposeObject(new THREE.Mesh(geometry)); + + expect(onDispose).toHaveBeenCalledTimes(1); + }); + + it("stops shader ticks for disposed water but keeps the rest", () => { + const scene = makeScene(); + const water = waterLikeMesh(); + const otherWater = waterLikeMesh(); + scene.shader_objects = [water, otherWater] as never; + const group = new THREE.Group(); + group.add(water); + + scene.disposeObject(group); + + expect(scene.shader_objects).toEqual([otherWater]); + }); + + it("stops and forgets the video behind a video plane", () => { + const scene = makeScene(); + const video = stubbedVideo(); + const otherVideo = stubbedVideo(); + scene.video_planes = [video, otherVideo]; + const plane = new THREE.Mesh( + new THREE.PlaneGeometry(), + new THREE.MeshBasicMaterial({ map: new THREE.VideoTexture(video) }), + ); + + scene.disposeObject(plane); + + expect(video.pause).toHaveBeenCalled(); + expect(video.load).toHaveBeenCalled(); + expect(video.hasAttribute("src")).toBe(false); + expect(scene.video_planes).toEqual([otherVideo]); + expect(otherVideo.pause).not.toHaveBeenCalled(); + }); + + it("removes MMD meshes from the animation helper", () => { + const scene = makeScene(); + const mmd = new THREE.SkinnedMesh(); + const other = new THREE.SkinnedMesh(); + const remove = vi.fn(); + scene.helper = { meshes: [mmd, other], remove } as never; + + scene.disposeObject(mmd); + + expect(remove).toHaveBeenCalledTimes(1); + expect(remove).toHaveBeenCalledWith(mmd); + }); + }); + + describe("disposeContents", () => { + it("empties the scene and frees every child", () => { + const scene = makeScene(); + const geometries = [new THREE.BoxGeometry(), new THREE.SphereGeometry()]; + const disposed = new Set(); + for (const geometry of geometries) { + geometry.addEventListener("dispose", () => disposed.add(geometry)); + scene.scene.add(new THREE.Mesh(geometry)); + } + const child = scene.scene.children[0]; + scene.hot_items = [child]; + + scene.disposeContents(); + + expect(scene.scene.children).toHaveLength(0); + expect(child.parent).toBeNull(); + expect(scene.hot_items).toEqual([]); + expect(disposed).toEqual(new Set(geometries)); + }); + + it("leaves the background for the skybox to replace", () => { + const scene = makeScene(); + const background = new THREE.CubeTexture(); + const onDispose = vi.fn(); + background.addEventListener("dispose", onDispose); + scene.scene.background = background; + + scene.disposeContents(); + + expect(scene.scene.background).toBe(background); + expect(onDispose).not.toHaveBeenCalled(); + }); + }); + + it("dispose frees the contents and the background", () => { + const scene = makeScene(); + const geometry = new THREE.BoxGeometry(); + const background = new THREE.CubeTexture(); + const disposed = new Set(); + geometry.addEventListener("dispose", () => disposed.add(geometry)); + background.addEventListener("dispose", () => disposed.add(background)); + scene.scene.add(new THREE.Mesh(geometry)); + scene.scene.background = background; + + scene.dispose(); + + expect(disposed).toEqual(new Set([geometry, background])); + expect(scene.scene.children).toHaveLength(0); + expect(scene.scene.background).toBeNull(); + }); + + it("disposeBackground frees a texture background", () => { + const scene = makeScene(); + const background = new THREE.CubeTexture(); + const onDispose = vi.fn(); + background.addEventListener("dispose", onDispose); + scene.scene.background = background; + + scene.disposeBackground(); + + expect(onDispose).toHaveBeenCalledTimes(1); + }); +}); + +function makeScene(): Scene { + return new Scene("test", "::CAM::", () => undefined, 1, { + getCameras: () => [], + getSelectedCameraId: () => "main", + fetchAsset: () => Promise.reject(new Error("no network in tests")), + getMediaUrlByToken: () => Promise.reject(new Error("no network in tests")), + }); +} + +// Scene only tracks Water instances by identity; a plain mesh stands in. +function waterLikeMesh(): THREE.Mesh { + return new THREE.Mesh( + new THREE.PlaneGeometry(), + new THREE.MeshBasicMaterial(), + ); +} + +// jsdom does not implement media playback, so stub the calls stopVideo makes. +function stubbedVideo(): HTMLVideoElement { + const video = document.createElement("video"); + video.src = "https://example.invalid/clip.mp4"; + video.pause = vi.fn(); + video.load = vi.fn(); + return video; +} diff --git a/frontend/libs/components/pagescene/src/lib/engine/scene.ts b/frontend/libs/components/pagescene/src/lib/engine/scene.ts index cf5972e565..83fbae813d 100644 --- a/frontend/libs/components/pagescene/src/lib/engine/scene.ts +++ b/frontend/libs/components/pagescene/src/lib/engine/scene.ts @@ -13,6 +13,10 @@ import type { Camera } from "@storyteller/common"; import toast from "react-hot-toast"; import { SplatMesh } from "@sparkjsdev/spark"; import { ensureInternalBbox } from "./internalBbox"; +import { + collectObjectTextures, + disposeObject3D, +} from "@storyteller/ui-viewer-3d"; // Capabilities Scene needs from outside its own state. Editor wires // these in inline at construction (Phase 2 idiom — same shape as @@ -134,16 +138,64 @@ class Scene { } clear() { - this.scene.children = []; - // hot_items held detached keyframe-point refs across reloads; reset it - // so it never carries stale objects from a previous scene. - this.hot_items = []; + this.disposeContents(); this._createGrid(); this._create_base_lighting(); this._create_skybox(); this._create_camera_obj(); } + // Remove everything from the scene and free its GPU resources. The + // background is left in place: _create_skybox frees it when it is + // replaced, and teardown frees it through disposeBackground. + disposeContents() { + this.disposeObject(this.scene); + this.scene.clear(); + // hot_items held detached keyframe-point refs across reloads; reset it + // so it never carries stale objects from a previous scene. + this.hot_items = []; + } + + // Free an object tree that has left the scene for good: its GPU + // resources, plus its entries in the per-object registries (Water + // shaders ticked every frame, video elements, MMD animation). + disposeObject(root: THREE.Object3D) { + const inTree = new Set(); + root.traverse((child) => inTree.add(child)); + this.shader_objects = this.shader_objects.filter( + (water) => !inTree.has(water), + ); + for (const mesh of [...(this.helper?.meshes ?? [])]) { + if (inTree.has(mesh)) this.helper.remove(mesh); + } + + const videos = new Set(); + for (const texture of collectObjectTextures(root)) { + if (texture instanceof THREE.VideoTexture) videos.add(texture.image); + } + for (const video of videos) stopVideo(video); + this.video_planes = this.video_planes.filter( + (video) => !videos.has(video), + ); + + disposeObject3D(root); + } + + // Final teardown when the owning Editor unmounts: free the contents and + // the background. The Scene is not used again — every mount builds a new + // Editor, and EngineProvider serializes the scene before unmounting. + dispose() { + this.disposeContents(); + this.disposeBackground(); + this.scene.background = null; + } + + disposeBackground() { + if (this.scene.background instanceof THREE.Texture) { + this.scene.background.dispose(); + } + } + async instantiate( name: string, pos: THREE.Vector3 = new THREE.Vector3(0, 0, 0), @@ -1118,6 +1170,9 @@ class Scene { // default skybox. _create_skybox() { const loader = new THREE.CubeTextureLoader(); + // Every branch below replaces the background, so free the cube texture + // being replaced (skybox switches and New Scene used to leak it). + this.disposeBackground(); // Theme-aware: in light mode, use a near-white background for better contrast with UI try { @@ -1272,4 +1327,12 @@ class Scene { } +// Pausing alone keeps the decoder and buffered media alive; dropping the +// source and reloading is what releases them. +function stopVideo(video: HTMLVideoElement) { + video.pause(); + video.removeAttribute("src"); + video.load(); +} + export default Scene; diff --git a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.spec.ts b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.spec.ts new file mode 100644 index 0000000000..fbb7123613 --- /dev/null +++ b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.spec.ts @@ -0,0 +1,91 @@ +import * as THREE from "three"; +import Scene from "../engine/scene"; +import type { LoadTicket } from "../engine/save_manager"; +import { StoryTellerProxyScene } from "./storyteller_proxy_scene"; + +vi.mock("@sparkjsdev/spark", async () => { + const { Object3D } = await import("three"); + return { + SplatMesh: class extends Object3D {}, + SparkRenderer: class extends Object3D {}, + }; +}); + +// The Addons barrel pulls in a lottie canvas module jsdom cannot run. +vi.mock("three/examples/jsm/Addons.js", async () => { + const { Mesh } = await import("three"); + return { MMDAnimationHelper: class {}, Water: class extends Mesh {} }; +}); + +const BOX_JSON = { + media_file_token: "Parim", + object_name: "Box", + object_user_data_name: "Box", + object_uuid: "box-uuid", + position: new THREE.Vector3(), + rotation: new THREE.Vector3(), + scale: new THREE.Vector3(1, 1, 1), + color: "#FFFFFF", + metalness: 0, + shininess: 0.5, + specular: 0, + locked: false, + visible: true, + media_file_type: "none", + user_data: { shapeKey: "Box" }, +}; + +describe("loadFromSceneJson", () => { + it("removes and frees objects that finish after the load is cancelled", async () => { + const scene = makeScene(); + const proxy = new StoryTellerProxyScene(2, scene); + const ticket = makeTicket(); + const onDispose = vi.fn(); + const addListener = trackFirstMeshGeometry(scene, onDispose); + + const load = proxy.loadFromSceneJson([BOX_JSON] as never, "m_1", 2, ticket); + // The box is created before the loader's first await on its tasks; + // cancel while that await is pending, as a newer load or an unmount + // would. + addListener(); + ticket.cancelled = true; + await load; + + expect(scene.scene.getObjectByProperty("name", "Box")).toBeUndefined(); + expect(onDispose).toHaveBeenCalledTimes(1); + }); + + it("keeps the loaded objects when the load is not cancelled", async () => { + const scene = makeScene(); + const proxy = new StoryTellerProxyScene(2, scene); + + await proxy.loadFromSceneJson([BOX_JSON] as never, "m_1", 2, makeTicket()); + + expect(scene.get_object_by_uuid("box-uuid")).toBeDefined(); + }); +}); + +function makeScene(): Scene { + return new Scene("test", "::CAM::", () => undefined, 2, { + getCameras: () => [], + getSelectedCameraId: () => "main", + fetchAsset: () => Promise.reject(new Error("no network in tests")), + getMediaUrlByToken: () => Promise.reject(new Error("no network in tests")), + }); +} + +function makeTicket(): LoadTicket { + return { cancelled: false, signal: new AbortController().signal }; +} + +// Returns a function that, once the load has created its box, subscribes +// `onDispose` to that box's geometry. +function trackFirstMeshGeometry( + scene: Scene, + onDispose: () => void, +): () => void { + return () => { + const box = scene.scene.getObjectByProperty("name", "Box") as THREE.Mesh; + box.geometry.addEventListener("dispose", onDispose); + }; +} diff --git a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts index 225a7d790e..4debc117f4 100644 --- a/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts +++ b/frontend/libs/components/pagescene/src/lib/proxy/storyteller_proxy_scene.ts @@ -128,9 +128,9 @@ export class StoryTellerProxyScene { ) { if (scene_json == null || this.scene == null) return; - while (this.scene.scene.children.length > 0) { - this.scene.scene.remove(this.scene.scene.children[0]); - } + // Free the outgoing scene before loading over it; removing the + // children alone left all of their GPU resources allocated. + this.scene.disposeContents(); // Warm Scene's URL cache up front so per-asset loadObject() calls // never hit the network for token→URL resolution. One batch call @@ -229,7 +229,10 @@ export class StoryTellerProxyScene { ); const settled = await Promise.allSettled(tasks); - if (ticket?.cancelled) return; + if (ticket?.cancelled) { + this.discardLoadedObjects(settled); + return; + } // Synchronous transform-application pass. We walk results in // original JSON order to preserve scene.children insertion @@ -295,4 +298,21 @@ export class StoryTellerProxyScene { this.scene._createGrid(); this.scene.updateSkybox(skybox_media_id); } + + // A cancelled load has still added whatever finished loading to the + // scene. Take those objects back out and free them, or they would leak + // and, when a newer load superseded this one, show up in its scene. + // Lights are skipped: Scene tracks its base lights in fields that a + // newer load may have re-pointed at them. + private discardLoadedObjects( + settled: PromiseSettledResult<{ obj: THREE.Object3D | undefined }>[], + ) { + for (const result of settled) { + if (result.status !== "fulfilled") continue; + const { obj } = result.value; + if (!obj || obj instanceof THREE.Light) continue; + obj.removeFromParent(); + this.scene.disposeObject(obj); + } + } } diff --git a/frontend/libs/components/pagescene/tsconfig.lib.json b/frontend/libs/components/pagescene/tsconfig.lib.json index f7a21e4eb9..28460cea8e 100644 --- a/frontend/libs/components/pagescene/tsconfig.lib.json +++ b/frontend/libs/components/pagescene/tsconfig.lib.json @@ -124,6 +124,9 @@ }, { "path": "../../icons/tsconfig.lib.json" + }, + { + "path": "../viewer-3d/tsconfig.lib.json" } ] } diff --git a/frontend/libs/components/upload-modal/src/lib/UploadModal3D/UploadFiles3D.tsx b/frontend/libs/components/upload-modal/src/lib/UploadModal3D/UploadFiles3D.tsx index 58c3b74552..dd2efe88ae 100644 --- a/frontend/libs/components/upload-modal/src/lib/UploadModal3D/UploadFiles3D.tsx +++ b/frontend/libs/components/upload-modal/src/lib/UploadModal3D/UploadFiles3D.tsx @@ -160,12 +160,13 @@ export const UploadFiles3D = ({ }); }; + // Tears down the current preview: stops its loop, cancels in-flight + // loads and frees its scene and renderer (see loadPreviewOnCanvas). + const disposePreviewRef = useRef<(() => void) | null>(null); const disposeRenderer = () => { - if (rendererRef.current) { - rendererRef.current.setAnimationLoop(null); - rendererRef.current.dispose(); - rendererRef.current = null; - } + disposePreviewRef.current?.(); + disposePreviewRef.current = null; + rendererRef.current = null; }; // Normalize an FBX to GLB in the background. Completion swaps the entry's @@ -237,7 +238,7 @@ export const UploadFiles3D = ({ return; } - const { renderer, camera, selectAnimation, setSkeletonVisible, cancel } = + const { renderer, camera, selectAnimation, setSkeletonVisible, dispose } = loadPreviewOnCanvas({ file: currentFile, canvas: canvasRef.current, @@ -250,17 +251,15 @@ export const UploadFiles3D = ({ }, }); rendererRef.current = renderer; + disposePreviewRef.current = dispose; cameraRef.current = camera; selectAnimationRef.current = selectAnimation; setSkeletonVisibleRef.current = setSkeletonVisible; - return () => { - // Cancel BEFORE disposing: an in-flight load's callbacks must not - // render on the disposed renderer or write into the next preview's - // status/animation/rig state (or its thumbnail). - cancel(); - disposeRenderer(); - }; + // dispose() cancels first: an in-flight load's callbacks must not + // render on the disposed renderer or write into the next preview's + // status/animation/rig state (or its thumbnail). + return disposeRenderer; // eslint-disable-next-line react-hooks/exhaustive-deps }, [previewIndex, filesVersion]); diff --git a/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.spec.ts b/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.spec.ts new file mode 100644 index 0000000000..4e97bd7981 --- /dev/null +++ b/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.spec.ts @@ -0,0 +1,124 @@ +import * as THREE from "three"; +import { loadPreviewOnCanvas } from "./loadPreviewOnCanvas"; + +const { renderers, pendingGltfLoads } = vi.hoisted(() => ({ + renderers: [] as Array<{ + dispose: ReturnType; + forceContextLoss: ReturnType; + setAnimationLoop: ReturnType; + }>, + pendingGltfLoads: [] as Array<(gltf: { scene: unknown }) => void>, +})); + +// jsdom has no WebGL; only the lifecycle calls matter here. +vi.mock("three", async (importOriginal) => { + const actual = await importOriginal(); + class WebGLRenderer { + dispose = vi.fn(); + forceContextLoss = vi.fn(); + setAnimationLoop = vi.fn(); + constructor() { + renderers.push(this); + } + setSize() {} + render() {} + } + return { ...actual, WebGLRenderer }; +}); + +// Loads stay pending until a test resolves them, to model a load that +// finishes after the preview was superseded. +vi.mock("three/addons/loaders/GLTFLoader.js", () => ({ + GLTFLoader: class { + load(_url: string, onLoad: (gltf: { scene: unknown }) => void) { + pendingGltfLoads.push(onLoad); + } + }, +})); + +vi.mock("@sparkjsdev/spark", async () => { + const { Object3D } = await import("three"); + return { + SplatFileType: { SPZ: "spz" }, + SplatMesh: class extends Object3D {}, + SparkRenderer: class extends Object3D {}, + }; +}); + +describe("loadPreviewOnCanvas dispose", () => { + beforeEach(() => { + renderers.length = 0; + pendingGltfLoads.length = 0; + vi.stubGlobal("URL", { + ...URL, + createObjectURL: vi.fn(() => "blob:preview"), + revokeObjectURL: vi.fn(), + }); + }); + + it("stops the loop, frees the renderer and revokes the blob URL", () => { + const canvas = attachedCanvas(); + const preview = loadPreviewOnCanvas({ + file: new File([], "model.glb"), + canvas, + statusCallback: vi.fn(), + }); + + preview.dispose(); + + const renderer = renderers[0]; + expect(renderer.setAnimationLoop).toHaveBeenLastCalledWith(null); + expect(renderer.dispose).toHaveBeenCalledTimes(1); + expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:preview"); + }); + + it("keeps the context while the canvas is reused for the next file", () => { + const preview = loadPreviewOnCanvas({ + file: new File([], "model.glb"), + canvas: attachedCanvas(), + statusCallback: vi.fn(), + }); + + preview.dispose(); + + expect(renderers[0].forceContextLoss).not.toHaveBeenCalled(); + }); + + it("releases the context once the canvas has left the DOM", () => { + const canvas = attachedCanvas(); + const preview = loadPreviewOnCanvas({ + file: new File([], "model.glb"), + canvas, + statusCallback: vi.fn(), + }); + + canvas.remove(); + preview.dispose(); + + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); + + it("ignores a model that finishes loading after dispose", () => { + const statusCallback = vi.fn(); + const preview = loadPreviewOnCanvas({ + file: new File([], "model.glb"), + canvas: attachedCanvas(), + statusCallback, + }); + + preview.dispose(); + const lateModel = new THREE.Group(); + lateModel.add(new THREE.Mesh(new THREE.BoxGeometry())); + pendingGltfLoads[0]({ scene: lateModel }); + + expect(lateModel.children).toHaveLength(1); + expect(statusCallback).not.toHaveBeenCalled(); + }); +}); + +function attachedCanvas(): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.getContext = vi.fn(() => null) as never; + document.body.appendChild(canvas); + return canvas; +} diff --git a/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.ts b/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.ts index b1eba45872..d3209c218f 100644 --- a/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.ts +++ b/frontend/libs/components/upload-modal/src/lib/UploadModal3D/utilities/loadPreviewOnCanvas.ts @@ -3,10 +3,12 @@ import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; import { FontLoader } from "three/addons/loaders/FontLoader.js"; import { TextGeometry } from "three/addons/geometries/TextGeometry.js"; import { SplatFileType, SplatMesh } from "@sparkjsdev/spark"; -import { NodeHierarchyHelper } from "@storyteller/ui-viewer-3d"; +import { NodeHierarchyHelper, disposeObject3D } from "@storyteller/ui-viewer-3d"; interface LoaderInterface { file: File; + // Blob URL for `file`, revoked when the preview is disposed. + fileUrl: string; camera: THREE.PerspectiveCamera; scene: THREE.Scene; renderer: THREE.WebGLRenderer; @@ -31,9 +33,10 @@ interface PreviewReturn { // Show/hide the skeleton overlay (rigged GLBs only; no-op otherwise). // Defaults visible for mesh-less models so there's something to see. setSkeletonVisible: (visible: boolean) => void; - // Call when a newer preview supersedes this one (effect cleanup): any - // still-in-flight load's callbacks become no-ops. - cancel: () => void; + // Call when a newer preview supersedes this one or the preview goes away + // (effect cleanup): in-flight loads become no-ops, the animation loop + // stops, and the scene, renderer and blob URL are freed. + dispose: () => void; } export const loadPreviewOnCanvas = ({ @@ -83,6 +86,7 @@ export const loadPreviewOnCanvas = ({ scene.add(light); let splatMesh: SplatMesh | null = null; + const fileUrl = URL.createObjectURL(file); // Supersession flag: each preview invocation owns one. The effect that // created this preview flips it in its cleanup, and every async loader @@ -90,9 +94,6 @@ export const loadPreviewOnCanvas = ({ // renderer, write status/animation/rig state into the NEXT preview, or // trigger a thumbnail snapshot of the wrong model. let cancelled = false; - const cancel = () => { - cancelled = true; - }; const isCancelled = () => cancelled; // Animation playback state for GLBs with baked clips. The mixer is rooted @@ -143,6 +144,7 @@ export const loadPreviewOnCanvas = ({ if (file.name.endsWith(".glb")) { glbLoader({ file, + fileUrl, scene, camera, renderer, @@ -181,7 +183,15 @@ export const loadPreviewOnCanvas = ({ }, }); } else if (file.name.endsWith(".pmd")) { - pmdLoader({ file, scene, camera, renderer, statusCallback, isCancelled }); + pmdLoader({ + file, + fileUrl, + scene, + camera, + renderer, + statusCallback, + isCancelled, + }); } else if ( file.name.endsWith(".png") || file.name.endsWith(".jpg") || @@ -190,6 +200,7 @@ export const loadPreviewOnCanvas = ({ ) { imagePlaneLoader({ file, + fileUrl, scene, camera, renderer, @@ -242,11 +253,26 @@ export const loadPreviewOnCanvas = ({ }; renderer.setAnimationLoop(animate); - return { renderer, camera, selectAnimation, setSkeletonVisible, cancel }; + // The canvas is reused for the next file's preview, which hands that + // renderer this same WebGL context, so the context loss is forced only + // once the canvas has left the DOM (the modal closed). + const dispose = () => { + cancelled = true; + renderer.setAnimationLoop(null); + disposeObject3D(scene); + // A splat still decoding when the preview was superseded never made it + // into the scene. + if (splatMesh && !splatMesh.parent) splatMesh.dispose(); + URL.revokeObjectURL(fileUrl); + renderer.dispose(); + if (!canvas.isConnected) renderer.forceContextLoss(); + }; + + return { renderer, camera, selectAnimation, setSkeletonVisible, dispose }; }; const glbLoader = ({ - file, + fileUrl, camera, scene, renderer, @@ -257,7 +283,7 @@ const glbLoader = ({ }: LoaderInterface) => { const loader = new GLTFLoader(); loader.load( - URL.createObjectURL(file), + fileUrl, (data) => { if (isCancelled?.()) return; // Inspect BEFORE re-parenting (the loop below moves children out of @@ -403,7 +429,7 @@ const pmdLoader = ({ }; const imagePlaneLoader = ({ - file, + fileUrl, scene, statusCallback, isCancelled, @@ -411,7 +437,7 @@ const imagePlaneLoader = ({ const geometry = new THREE.PlaneGeometry(1, 1); const loader = new THREE.TextureLoader(); const texture = loader.load( - URL.createObjectURL(file), + fileUrl, undefined, undefined, (loaderError) => { diff --git a/frontend/libs/components/upload-modal/src/lib/UploadModalSplat/UploadFilesSplat.tsx b/frontend/libs/components/upload-modal/src/lib/UploadModalSplat/UploadFilesSplat.tsx index ad5dd639ae..dbf4020293 100644 --- a/frontend/libs/components/upload-modal/src/lib/UploadModalSplat/UploadFilesSplat.tsx +++ b/frontend/libs/components/upload-modal/src/lib/UploadModalSplat/UploadFilesSplat.tsx @@ -92,12 +92,13 @@ export const UploadFilesSplat = ({ const [overallProgress, setOverallProgress] = useState<{ current: number; total: number } | null>(null); const [selectionError, setSelectionError] = useState(); + // Tears down the current preview: stops its loop, cancels in-flight + // loads and frees its scene and renderer (see loadPreviewOnCanvas). + const disposePreviewRef = useRef<(() => void) | null>(null); const disposeRenderer = () => { - if (rendererRef.current) { - rendererRef.current.setAnimationLoop(null); - rendererRef.current.dispose(); - rendererRef.current = null; - } + disposePreviewRef.current?.(); + disposePreviewRef.current = null; + rendererRef.current = null; }; useEffect(() => { @@ -107,12 +108,13 @@ export const UploadFilesSplat = ({ disposeRenderer(); setPreviewStatus({ type: "init" }); - const { renderer, camera } = loadPreviewOnCanvas({ + const { renderer, camera, dispose } = loadPreviewOnCanvas({ file: currentFile, canvas: canvasRef.current, statusCallback: setPreviewStatus, }); rendererRef.current = renderer; + disposePreviewRef.current = dispose; cameraRef.current = camera; return disposeRenderer; diff --git a/frontend/libs/components/viewer-3d/src/index.ts b/frontend/libs/components/viewer-3d/src/index.ts index c52bc6084f..fcbaf013bb 100644 --- a/frontend/libs/components/viewer-3d/src/index.ts +++ b/frontend/libs/components/viewer-3d/src/index.ts @@ -1,3 +1,4 @@ export * from './lib/viewer-3d'; export * from './lib/NodeHierarchyHelper'; +export * from './lib/disposeObject3D'; diff --git a/frontend/libs/components/viewer-3d/src/lib/disposeObject3D.spec.ts b/frontend/libs/components/viewer-3d/src/lib/disposeObject3D.spec.ts new file mode 100644 index 0000000000..0a1066c30e --- /dev/null +++ b/frontend/libs/components/viewer-3d/src/lib/disposeObject3D.spec.ts @@ -0,0 +1,160 @@ +import * as THREE from "three"; +import { SparkRenderer, SplatMesh } from "@sparkjsdev/spark"; +import { disposeObject3D } from "./disposeObject3D"; + +// A real SplatMesh / SparkRenderer needs Spark's WASM decoder and a WebGL +// renderer; only `instanceof` and the dispose() calls matter here. +vi.mock("@sparkjsdev/spark", async () => { + const { Object3D } = await import("three"); + const accumulator = () => ({ splats: { dispose: vi.fn() } }); + return { + SplatMesh: class extends Object3D { + dispose = vi.fn(); + }, + SparkRenderer: class extends Object3D { + defaultView = { dispose: vi.fn() }; + active = accumulator(); + freeAccumulators = [accumulator(), accumulator()]; + }, + }; +}); + +describe("disposeObject3D", () => { + it("disposes nested geometries, materials and their textures", () => { + const map = new THREE.Texture(); + const normalMap = new THREE.Texture(); + const material = new THREE.MeshStandardMaterial({ map, normalMap }); + const geometry = new THREE.BoxGeometry(); + const root = new THREE.Group(); + const child = new THREE.Group(); + child.add(new THREE.Mesh(geometry, material)); + root.add(child); + + const disposed = trackDisposals([map, normalMap, material, geometry]); + disposeObject3D(root); + + expect(disposed).toEqual(new Set([map, normalMap, material, geometry])); + }); + + it("disposes every material of a multi-material mesh", () => { + const first = new THREE.MeshBasicMaterial({ map: new THREE.Texture() }); + const second = new THREE.MeshBasicMaterial(); + const mesh = new THREE.Mesh(new THREE.BoxGeometry(), [first, second]); + + const disposed = trackDisposals([first, second, first.map!]); + disposeObject3D(mesh); + + expect(disposed).toEqual(new Set([first, second, first.map])); + }); + + it("disposes a material shared by several meshes once", () => { + const material = new THREE.MeshBasicMaterial(); + const root = new THREE.Group(); + root.add(new THREE.Mesh(new THREE.BoxGeometry(), material)); + root.add(new THREE.Mesh(new THREE.BoxGeometry(), material)); + + const onDispose = vi.fn(); + material.addEventListener("dispose", onDispose); + disposeObject3D(root); + + expect(onDispose).toHaveBeenCalledTimes(1); + }); + + it("disposes skinned mesh skeletons", () => { + const bone = new THREE.Bone(); + const skeleton = new THREE.Skeleton([bone]); + const mesh = new THREE.SkinnedMesh( + new THREE.BoxGeometry(), + new THREE.MeshBasicMaterial(), + ); + mesh.add(bone); + mesh.bind(skeleton); + const dispose = vi.spyOn(skeleton, "dispose"); + + disposeObject3D(mesh); + + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it("frees splat buffers", () => { + const splat = new SplatMesh(); + const root = new THREE.Group(); + root.add(splat); + + disposeObject3D(root); + + expect(splat.dispose).toHaveBeenCalledTimes(1); + }); + + it("frees a SparkRenderer's viewpoint and accumulators", () => { + const spark = new SparkRenderer({} as never); + const root = new THREE.Group(); + root.add(spark); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { freeAccumulators } = spark as any; + + disposeObject3D(root); + + expect(spark.defaultView.dispose).toHaveBeenCalledTimes(1); + expect(spark.active.splats.dispose).toHaveBeenCalledTimes(1); + for (const accumulator of freeAccumulators) { + expect(accumulator.splats.dispose).toHaveBeenCalledTimes(1); + } + }); + + it("disposes textures held in ShaderMaterial uniforms", () => { + const normals = new THREE.Texture(); + const mirror = new THREE.WebGLRenderTarget(4, 4).texture; + const layers = [new THREE.Texture(), new THREE.Texture()]; + const material = new THREE.ShaderMaterial({ + uniforms: { + normalSampler: { value: normals }, + mirrorSampler: { value: mirror }, + layerSamplers: { value: layers }, + size: { value: 1.0 }, + }, + }); + const mesh = new THREE.Mesh(new THREE.PlaneGeometry(), material); + + const disposed = trackDisposals([normals, mirror, ...layers]); + disposeObject3D(mesh); + + expect(disposed).toEqual(new Set([normals, mirror, ...layers])); + }); + + it("disposes a texture shared by a property and a uniform once", () => { + const video = new THREE.Texture(); + const material = new THREE.ShaderMaterial({ + uniforms: { map: { value: video } }, + }); + // ChromaKeyMaterial keeps its VideoTexture both here and in a uniform. + (material as unknown as { texture: THREE.Texture }).texture = video; + const mesh = new THREE.Mesh(new THREE.PlaneGeometry(), material); + + const onDispose = vi.fn(); + video.addEventListener("dispose", onDispose); + disposeObject3D(mesh); + + expect(onDispose).toHaveBeenCalledTimes(1); + }); + + it("skips objects without geometry or material", () => { + const root = new THREE.Group(); + root.add(new THREE.DirectionalLight()); + root.add(new THREE.PerspectiveCamera()); + + expect(() => disposeObject3D(root)).not.toThrow(); + }); +}); + +// Records which resources dispatched their "dispose" event. +function trackDisposals( + resources: Array, +): Set { + const disposed = new Set(); + for (const resource of resources) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (resource as any).addEventListener("dispose", () => disposed.add(resource)); + } + return disposed; +} diff --git a/frontend/libs/components/viewer-3d/src/lib/disposeObject3D.ts b/frontend/libs/components/viewer-3d/src/lib/disposeObject3D.ts new file mode 100644 index 0000000000..e468c5e5b1 --- /dev/null +++ b/frontend/libs/components/viewer-3d/src/lib/disposeObject3D.ts @@ -0,0 +1,93 @@ +import * as THREE from "three"; +import { SparkRenderer, SplatMesh } from "@sparkjsdev/spark"; + +// Free the GPU resources held by an object tree that is leaving the scene +// for good: geometries, materials, the textures those materials reference +// (Material.dispose() does not free its maps), skeleton bone textures and +// Spark splat buffers. Call it after the tree has been removed. +// +// Textures are found both as material properties (map, normalMap, ...) and +// as ShaderMaterial uniforms (Water's normal and mirror samplers, Spark's +// splat textures). Water keeps its mirror render target in a closure, so +// only that target's color texture can be reached; its framebuffer and depth +// buffer stay allocated. +// +// Geometry, material and texture disposal is safe even if one of them is +// shared with an object that stays — three.js re-uploads it the next time +// it renders. Splat buffers are not re-creatable, but every SplatMesh owns +// its own (scene.ts builds one per load), and undo re-loads deleted +// objects from their snapshot rather than reusing the instance. +export function disposeObject3D(root: THREE.Object3D): void { + root.traverse((child) => { + if (child instanceof SplatMesh) child.dispose(); + if (child instanceof SparkRenderer) disposeSparkRenderer(child); + if (child instanceof THREE.SkinnedMesh) child.skeleton?.dispose(); + (child as Partial).geometry?.dispose(); + }); + + const materials = collectMaterials(root); + const textures = collectMaterialTextures(materials); + for (const material of materials) material.dispose(); + for (const texture of textures) texture.dispose(); +} + +// Every texture the tree's materials reference, as material properties or +// ShaderMaterial uniforms, deduplicated. +export function collectObjectTextures( + root: THREE.Object3D, +): Set { + return collectMaterialTextures(collectMaterials(root)); +} + +// A SparkRenderer is added to the scene automatically the first time a +// splat renders. Its viewpoint render targets and splat accumulators are +// not reachable from its material, so free them explicitly. The viewpoint +// goes first: disposing it returns its accumulators to the free list. +function disposeSparkRenderer(spark: SparkRenderer): void { + spark.defaultView.dispose(); + spark.active.splats.dispose(); + const { freeAccumulators } = spark as unknown as { + freeAccumulators: SparkRenderer["active"][]; + }; + for (const accumulator of freeAccumulators) accumulator.splats.dispose(); +} + +function collectMaterials(root: THREE.Object3D): Set { + const materials = new Set(); + root.traverse((child) => { + const { material } = child as Partial; + const childMaterials = Array.isArray(material) ? material : [material]; + for (const childMaterial of childMaterials) { + if (childMaterial) materials.add(childMaterial); + } + }); + return materials; +} + +function collectMaterialTextures( + materials: Iterable, +): Set { + const textures = new Set(); + for (const material of materials) { + addTextures(Object.values(material), textures); + if (material instanceof THREE.ShaderMaterial) { + addTextures( + Object.values(material.uniforms).map((uniform) => uniform?.value), + textures, + ); + } + } + return textures; +} + +function addTextures(values: unknown[], into: Set): void { + for (const value of values) { + if (value instanceof THREE.Texture) { + into.add(value); + } else if (Array.isArray(value)) { + for (const item of value) { + if (item instanceof THREE.Texture) into.add(item); + } + } + } +} diff --git a/frontend/libs/components/viewer-3d/src/lib/viewer-3d.spec.tsx b/frontend/libs/components/viewer-3d/src/lib/viewer-3d.spec.tsx index c449c2683a..aac63e3f3b 100644 --- a/frontend/libs/components/viewer-3d/src/lib/viewer-3d.spec.tsx +++ b/frontend/libs/components/viewer-3d/src/lib/viewer-3d.spec.tsx @@ -2,11 +2,23 @@ import { render } from '@testing-library/react'; import StorytellerUiViewer3d from './viewer-3d'; +// jsdom has no ResizeObserver; the viewer only needs it to exist. +vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + disconnect() {} + }, +); + describe('StorytellerUiViewer3d', () => { - it('should render successfully', () => { const { baseElement } = render(); expect(baseElement).toBeTruthy(); }); - + + it('unmounts cleanly before the scene ever initialized', () => { + const { unmount } = render(); + expect(() => unmount()).not.toThrow(); + }); }); diff --git a/frontend/libs/components/viewer-3d/src/lib/viewer-3d.tsx b/frontend/libs/components/viewer-3d/src/lib/viewer-3d.tsx index a67043cc22..0ddde8f2ce 100644 --- a/frontend/libs/components/viewer-3d/src/lib/viewer-3d.tsx +++ b/frontend/libs/components/viewer-3d/src/lib/viewer-3d.tsx @@ -9,6 +9,7 @@ import { NodeHierarchyHelper, createRigHelper, } from "./NodeHierarchyHelper"; +import { disposeObject3D } from "./disposeObject3D"; // Dropdown value for the "no animation" choice (models rest in their bind / // T-pose). Clip values are their stringified index. @@ -277,20 +278,14 @@ export function Viewer3D({ ) { container.removeChild(rendererRef.current.domElement); } - rendererRef.current?.dispose(); controlsRef.current?.dispose(); - - if (cubeRef.current) { - const cube = cubeRef.current; - if (cube.geometry) cube.geometry.dispose(); - if (cube.material) { - if (Array.isArray(cube.material)) { - cube.material.forEach((m) => m.dispose()); - } else { - cube.material.dispose(); - } - } - } + // Free everything still in the scene (placeholder cube, grid, any + // model, Spark's auto-added splat renderer), then the context itself. + // The canvas is ours and already detached, so forcing the context + // loss frees it now instead of whenever the canvas is collected. + if (sceneRef.current) disposeObject3D(sceneRef.current); + rendererRef.current?.dispose(); + rendererRef.current?.forceContextLoss(); }; }, []); @@ -407,11 +402,19 @@ export function Viewer3D({ const SPLAT_EXTENSIONS = ["spz", "ply", "splat", "ksplat"]; const GLTF_EXTENSIONS = ["glb", "gltf", ""]; + // Set by this effect's cleanup. A model that finishes loading after the + // URL changed or the viewer unmounted is freed instead of shown. + let cancelled = false; + if (SPLAT_EXTENSIONS.includes(urlExtension)) { console.log(`[Viewer3D] splat format detected (.${urlExtension})`); new SplatMesh({ url: modelUrl, onLoad: (mesh) => { + if (cancelled) { + mesh.dispose(); + return; + } mesh.rotation.z = Math.PI; mesh.position.y = 1; scene.add(mesh); @@ -450,6 +453,10 @@ export function Viewer3D({ loader.load( modelUrl, (gltf) => { + if (cancelled) { + disposeObject3D(gltf.scene); + return; + } console.log("[Viewer3D] Model loaded successfully"); const model = gltf.scene; onModelLoaded(model); @@ -503,10 +510,12 @@ export function Viewer3D({ } return () => { + cancelled = true; stopAnimations(); removeSkeletonHelper(); - if (loadedModelRef.current && sceneRef.current) { - sceneRef.current.remove(loadedModelRef.current); + if (loadedModelRef.current) { + loadedModelRef.current.removeFromParent(); + disposeObject3D(loadedModelRef.current); loadedModelRef.current = null; } };