diff --git a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts index b38348ea84..ca1dfaf42e 100644 --- a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts +++ b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/ImageTo3DStore.ts @@ -2,145 +2,7 @@ import { create } from "zustand"; import { listen } from "@tauri-apps/api/event"; import { MediaFilesApi, MediaUploadApi } from "@storyteller/api"; import { v4 as uuidv4 } from "uuid"; -import * as THREE from "three"; -import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; - -// Offscreen thumbnail capture function - produces a square image -async function captureModelThumbnail(modelUrl: string): Promise { - return new Promise((resolve) => { - try { - // Square dimensions for thumbnail - const size = 512; - const fov = 50; - - const scene = new THREE.Scene(); - scene.background = new THREE.Color(0x282828); - - const camera = new THREE.PerspectiveCamera(fov, 1, 0.1, 1000); // Aspect ratio 1 for square - - const renderer = new THREE.WebGLRenderer({ - antialias: true, - alpha: false, - preserveDrawingBuffer: true, - }); - renderer.setSize(size, size); - renderer.setPixelRatio(1); - - // Add lights - const ambientLight = new THREE.AmbientLight(0xffffff, 2); - scene.add(ambientLight); - - const hemisphereLight = new THREE.HemisphereLight( - 0xffffff, - 0x888888, - 1.2, - ); - scene.add(hemisphereLight); - - const keyLight = new THREE.DirectionalLight(0xffffff, 2); - keyLight.position.set(2, 10, 8); - scene.add(keyLight); - - const fillLight = new THREE.DirectionalLight(0xffffff, 1.2); - fillLight.position.set(-6, 6, -4); - scene.add(fillLight); - - const frontLight = new THREE.DirectionalLight(0xffffff, 1); - frontLight.position.set(0, 4, 10); - scene.add(frontLight); - - const loader = new GLTFLoader(); - // Without anonymous CORS the GLTF + textures taint the WebGL canvas, so - // toDataURL() below throws a SecurityError and the thumbnail is lost. - loader.setCrossOrigin("anonymous"); - loader.load( - modelUrl, - (gltf) => { - const model = gltf.scene; - - const box = new THREE.Box3().setFromObject(model); - const modelSize = box.getSize(new THREE.Vector3()); - - const maxDim = Math.max(modelSize.x, modelSize.y, modelSize.z); - const scale = 2 / maxDim; - model.scale.multiplyScalar(scale); - - const scaledBox = new THREE.Box3().setFromObject(model); - const scaledCenter = scaledBox.getCenter(new THREE.Vector3()); - const scaledSize = scaledBox.getSize(new THREE.Vector3()); - - model.position.x = -scaledCenter.x; - model.position.z = -scaledCenter.z; - model.position.y = -scaledBox.min.y; - - scene.add(model); - - // Calculate camera distance to fit model in view with padding - const modelHeight = scaledSize.y; - const maxModelDim = Math.max( - scaledSize.x, - scaledSize.y, - scaledSize.z, - ); - - // Calculate distance needed to fit the model based on FOV - // For a 45-degree camera angle, we view from a diagonal - const fovRad = (fov * Math.PI) / 180; - const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2); - - // Add some padding (1.3x) and account for diagonal viewing angle - const cameraDistance = fitDistance * 1.3; - - // Position camera at 45-degree angle - const angle = Math.PI / 4; // 45 degrees - camera.position.set( - Math.sin(angle) * cameraDistance, - modelHeight * 0.5 + cameraDistance * 0.4, - Math.cos(angle) * cameraDistance, - ); - camera.lookAt(0, modelHeight * 0.4, 0); - - // Render and capture - renderer.render(scene, camera); - let dataUrl: string | null = null; - try { - dataUrl = renderer.domElement.toDataURL("image/png"); - } catch (err) { - // A tainted canvas (CORS) throws SecurityError here. Log so the - // failure is diagnosable instead of silently producing no thumbnail. - console.warn( - "[captureModelThumbnail] toDataURL failed (likely CORS taint):", - err, - ); - } - - // Cleanup - renderer.dispose(); - scene.clear(); - - resolve(dataUrl); - }, - undefined, - (error) => { - console.error("[captureModelThumbnail] Error loading model:", error); - renderer.dispose(); - scene.clear(); - resolve(null); - }, - ); - - // Timeout after 30 seconds - setTimeout(() => { - renderer.dispose(); - scene.clear(); - resolve(null); - }, 30000); - } catch (error) { - console.error("[captureModelThumbnail] Error:", error); - resolve(null); - } - }); -} +import { captureModelThumbnail } from "./captureModelThumbnail"; export interface PendingExternalImage { url: string; diff --git a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.spec.ts b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.spec.ts new file mode 100644 index 0000000000..b6345a6884 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.spec.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as THREE from "three"; +import { captureModelThumbnail } from "./captureModelThumbnail"; + +const { renderers, pendingLoads } = vi.hoisted(() => ({ + renderers: [] as Array<{ + dispose: ReturnType; + forceContextLoss: ReturnType; + }>, + pendingLoads: [] as Array<{ + onLoad: (gltf: { scene: THREE.Object3D }) => void; + onError: (error: unknown) => void; + }>, +})); + +// jsdom has no WebGL; only the teardown calls matter here. +vi.mock("three", async (importOriginal) => { + const actual = await importOriginal(); + class WebGLRenderer { + domElement = { toDataURL: () => "data:image/png;base64,AAAA" }; + dispose = vi.fn(); + forceContextLoss = vi.fn(); + constructor() { + renderers.push(this); + } + setSize() {} + setPixelRatio() {} + render() {} + } + return { ...actual, WebGLRenderer }; +}); + +// Loads stay pending until a test settles them. +vi.mock("three/examples/jsm/loaders/GLTFLoader.js", () => ({ + GLTFLoader: class { + setCrossOrigin() {} + load( + _url: string, + onLoad: (gltf: { scene: THREE.Object3D }) => void, + _onProgress: unknown, + onError: (error: unknown) => void, + ) { + pendingLoads.push({ onLoad, onError }); + } + }, +})); + +describe("captureModelThumbnail", () => { + beforeEach(() => { + renderers.length = 0; + pendingLoads.length = 0; + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("returns the frame and frees the model, renderer and context once", async () => { + const { model, onDispose } = trackedModel(); + const capture = captureModelThumbnail("model.glb"); + + pendingLoads[0].onLoad({ scene: model }); + const dataUrl = await capture; + vi.runAllTimers(); + + expect(dataUrl).toMatch(/^data:image\/png/); + expect(onDispose).toHaveBeenCalledTimes(1); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); + + it("frees the renderer and context when the model fails to load", async () => { + const capture = captureModelThumbnail("missing.glb"); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + pendingLoads[0].onError(new Error("404")); + + expect(await capture).toBeNull(); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + }); + + it("gives up after the timeout and frees a model that loads later", async () => { + const capture = captureModelThumbnail("slow.glb"); + + vi.advanceTimersByTime(30000); + expect(await capture).toBeNull(); + expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1); + + const { model, onDispose } = trackedModel(); + pendingLoads[0].onLoad({ scene: model }); + + expect(onDispose).toHaveBeenCalledTimes(1); + expect(renderers[0].dispose).toHaveBeenCalledTimes(1); + }); +}); + +function trackedModel() { + const geometry = new THREE.BoxGeometry(); + const onDispose = vi.fn(); + geometry.addEventListener("dispose", onDispose); + const model = new THREE.Group(); + model.add(new THREE.Mesh(geometry)); + return { model, onDispose }; +} diff --git a/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.ts b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.ts new file mode 100644 index 0000000000..d67d863d19 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageImageTo3DObject/captureModelThumbnail.ts @@ -0,0 +1,160 @@ +import * as THREE from "three"; +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; +import { disposeObject3D } from "@storyteller/ui-viewer-3d"; + +const THUMBNAIL_SIZE = 512; +const FOV = 50; +const LOAD_TIMEOUT_MS = 30000; + +// Offscreen thumbnail capture: loads the model into a throwaway renderer, +// renders one square frame and returns it as a PNG data URL (null on +// failure or timeout). The renderer, its WebGL context and the model are +// freed exactly once, whichever of load, error or timeout comes first. +export async function captureModelThumbnail( + modelUrl: string, +): Promise { + return new Promise((resolve) => { + try { + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x282828); + + const camera = new THREE.PerspectiveCamera(FOV, 1, 0.1, 1000); // Aspect ratio 1 for square + + const renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: false, + preserveDrawingBuffer: true, + }); + renderer.setSize(THUMBNAIL_SIZE, THUMBNAIL_SIZE); + renderer.setPixelRatio(1); + + addLights(scene); + + let done = false; + const finish = (dataUrl: string | null) => { + if (done) return; + done = true; + clearTimeout(timeout); + disposeObject3D(scene); + renderer.dispose(); + // The renderer's canvas is never attached, so nothing else can use + // this context; release it now instead of at garbage collection. + renderer.forceContextLoss(); + resolve(dataUrl); + }; + + const timeout = setTimeout(() => finish(null), LOAD_TIMEOUT_MS); + + const loader = new GLTFLoader(); + // Without anonymous CORS the GLTF + textures taint the WebGL canvas, so + // toDataURL() below throws a SecurityError and the thumbnail is lost. + loader.setCrossOrigin("anonymous"); + loader.load( + modelUrl, + (gltf) => { + // Timed out already: the renderer is gone, just free the model. + if (done) { + disposeObject3D(gltf.scene); + return; + } + const model = gltf.scene; + const modelHeight = fitModel(model); + scene.add(model); + frameCamera(camera, model, modelHeight); + + // Render and capture + renderer.render(scene, camera); + let dataUrl: string | null = null; + try { + dataUrl = renderer.domElement.toDataURL("image/png"); + } catch (err) { + // A tainted canvas (CORS) throws SecurityError here. Log so the + // failure is diagnosable instead of silently producing no thumbnail. + console.warn( + "[captureModelThumbnail] toDataURL failed (likely CORS taint):", + err, + ); + } + finish(dataUrl); + }, + undefined, + (error) => { + console.error("[captureModelThumbnail] Error loading model:", error); + finish(null); + }, + ); + } catch (error) { + console.error("[captureModelThumbnail] Error:", error); + resolve(null); + } + }); +} + +// Scale the model to a 2-unit max dimension and stand it centered on the +// origin. Returns its scaled height. +function fitModel(model: THREE.Object3D): number { + const box = new THREE.Box3().setFromObject(model); + const modelSize = box.getSize(new THREE.Vector3()); + + const maxDim = Math.max(modelSize.x, modelSize.y, modelSize.z); + const scale = 2 / maxDim; + model.scale.multiplyScalar(scale); + + const scaledBox = new THREE.Box3().setFromObject(model); + const scaledCenter = scaledBox.getCenter(new THREE.Vector3()); + + model.position.x = -scaledCenter.x; + model.position.z = -scaledCenter.z; + model.position.y = -scaledBox.min.y; + + return scaledBox.getSize(new THREE.Vector3()).y; +} + +// Place the camera on a 45-degree diagonal, far enough to fit the model. +function frameCamera( + camera: THREE.PerspectiveCamera, + model: THREE.Object3D, + modelHeight: number, +): void { + const scaledSize = new THREE.Box3() + .setFromObject(model) + .getSize(new THREE.Vector3()); + const maxModelDim = Math.max(scaledSize.x, scaledSize.y, scaledSize.z); + + // Calculate distance needed to fit the model based on FOV + // For a 45-degree camera angle, we view from a diagonal + const fovRad = (FOV * Math.PI) / 180; + const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2); + + // Add some padding (1.3x) and account for diagonal viewing angle + const cameraDistance = fitDistance * 1.3; + + // Position camera at 45-degree angle + const angle = Math.PI / 4; // 45 degrees + camera.position.set( + Math.sin(angle) * cameraDistance, + modelHeight * 0.5 + cameraDistance * 0.4, + Math.cos(angle) * cameraDistance, + ); + camera.lookAt(0, modelHeight * 0.4, 0); +} + +function addLights(scene: THREE.Scene): void { + const ambientLight = new THREE.AmbientLight(0xffffff, 2); + scene.add(ambientLight); + + const hemisphereLight = new THREE.HemisphereLight(0xffffff, 0x888888, 1.2); + scene.add(hemisphereLight); + + const keyLight = new THREE.DirectionalLight(0xffffff, 2); + keyLight.position.set(2, 10, 8); + scene.add(keyLight); + + const fillLight = new THREE.DirectionalLight(0xffffff, 1.2); + fillLight.position.set(-6, 6, -4); + scene.add(fillLight); + + const frontLight = new THREE.DirectionalLight(0xffffff, 1); + frontLight.position.set(0, 4, 10); + scene.add(frontLight); +}