Remove bug description, looks good

frontend/apps/editor3d/app/src/components/reusable/UploadModalSplat/UploadFilesSplat.tsx

@@ -8,6 +8,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faCube } from "@fortawesome/pro-solid-svg-icons";
 import { loadPreviewOnCanvas, snapshotCanvasAsThumbnail } from "../UploadModal3D/utilities";
 import { WebGLRenderer } from "three";
+import { upload3DObjects } from "../UploadModal3D/utilities/upload3DObjects";
 
 interface Props {
   title: string;
@@ -74,22 +75,22 @@ export const UploadFilesSplat = ({
     // Use the filename as the title
     const title = assetFile.value.name.split(".")[0];
 
-    const shouldFlip = title.endsWith("ceramic");
-
-    assetFile.value.arrayBuffer().then((arrayBuffer) => {
-      onLocalBytes(arrayBuffer, shouldFlip);
-    }).catch((error) => {
-      console.error("Error reading file as ArrayBuffer:", error);
-    });
-
-    // upload3DObjects({
-    //   title: title,
-    //   assetFile: assetFile.value,
-    //   thumbnailSnapshot: thumbnailFile,
-    //   engineCategory: engineCategory,
-    //   animationType: subtype,
-    //   progressCallback: onUploadProgress,
+    // const shouldFlip = title.search("ceramic") != -1;
+    //
+    // assetFile.value.arrayBuffer().then((arrayBuffer) => {
+    //   onLocalBytes(arrayBuffer, shouldFlip);
+    // }).catch((error) => {
+    //   console.error("Error reading file as ArrayBuffer:", error);
     // });
+
+    upload3DObjects({
+      title: title,
+      assetFile: assetFile.value,
+      thumbnailSnapshot: thumbnailFile,
+      engineCategory: engineCategory,
+      animationType: subtype,
+      progressCallback: onUploadProgress,
+    });
   };
 
   useEffect(() => {

frontend/apps/editor3d/app/src/enums/AssetType.ts

@@ -7,4 +7,5 @@ export enum AssetType {
   OBJECT = "object",
   SKYBOX = "skybox",
   SHAPE = "shape",
+  SPLAT = "splat",
 }

frontend/apps/editor3d/app/src/pages/PageEnigma/DragAndDrop/DndAsset.ts

@@ -96,24 +96,20 @@ class DndAsset {
         // if (dragItem.value.type === AssetType.CAMERA) {
         //   console.log("Dragged In Camera Type")
         // }
-        if (dragItem.value.type === AssetType.OBJECT) {
-          addObject(dragItem.value);
-        }
-
-        if (dragItem.value.type === AssetType.SHAPE) {
-          addShape(dragItem.value);
-        }
-
         /*
          FIXME:
          THIS IS A TEMPARARY SOLUTION TO A LONG PROBLEM WITH SKYBOXES
          UPDATE THIS WHEN UPLOADING SKYBOXES ARE FULLY IMPLEMENTED.
          THIS IS JUST TEMPARARY!!!
-        
         */
-        if (dragItem.value.type === AssetType.SKYBOX) {
-          addObject(dragItem.value);
+        if (mediaItem.type === AssetType.OBJECT || mediaItem.type === AssetType.SPLAT || mediaItem.type === AssetType.SKYBOX) {
+          addObject(mediaItem);
         }
+
+        if (mediaItem.type === AssetType.SHAPE) {
+          addShape(mediaItem);
+        }
+
         this.endDrag();
         return;
       }

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts

@@ -676,8 +676,8 @@ class Editor {
     this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
 
     // Add spark renderer as a child of the camera
-    this.activeScene.scene.add(this.sparkRenderer);
-    this.camera?.add(this.sparkRenderer);
+    // this.activeScene.scene.add(this.sparkRenderer);
+    // this.camera?.add(this.sparkRenderer);
 
     const onloadCallback = () => {
       console.log("Setting Scene is loaded");

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/scene.ts

@@ -17,6 +17,7 @@ import { InfiniteGridHelper } from "./InfiniteGridHelper";
 import { cameras, selectedCameraId } from "../signals/camera";
 import { MediaFilesApi } from "@storyteller/api";
 import toast from "react-hot-toast";
+import { SplatMesh } from "@sparkjsdev/spark";
 
 class Scene {
   name: string;
@@ -597,7 +598,17 @@ class Scene {
         obj.userData["media_file_type"] = MediaFileType.Image;
       }
       return obj;
+    } else if (
+      url.includes(".spz")
+    ) {
+      return await this.loadSplatWithPlaceholder(
+        media_id,
+        name,
+        auto_add,
+        position,
+      );
     }
+
     return await this.loadGlbWithPlaceholder(
       media_id,
       name,
@@ -629,6 +640,43 @@ class Scene {
     return false;
   }
 
+  async loadSplatWithPlaceholder(
+    media_id: string,
+    name: string,
+    auto_add: boolean = true,
+    position: THREE.Vector3 = new THREE.Vector3(-0.5, 1.5, 0),
+  ): Promise<THREE.Object3D> {
+    if (this.placeholder_manager === undefined) {
+      throw Error("Place holder Manager is undefined");
+    }
+
+    const url = await this.getMediaURL(media_id);
+    const key = media_id + name + generateUUID();
+    await this.placeholder_manager.add(key, `Loading: ${name}`, position);
+
+    const splat: SplatMesh = await this.load_splat_wrapped_no_cors(url, async () => {
+      if (this.placeholder_manager === undefined) {
+        throw Error("Place holder Manager is undefined");
+      }
+      await this.placeholder_manager.remove(key);
+    }).catch(async (error) => {
+      if (this.placeholder_manager !== undefined) {
+        await this.placeholder_manager.remove(key);
+      }
+      toast.error("Failed to load splat.");
+      throw error;
+    });
+
+    splat.userData["media_id"] = media_id;
+    splat.userData["media_file_type"] = MediaFileType.SPZ;
+
+    if (auto_add) {
+      this.scene.add(splat);
+    }
+    this.updateSurfaceIdAttributeToMesh(this.scene);
+    return splat;
+  }
+
   async loadGlbWithPlaceholder(
     media_id: string,
     name: string,
@@ -809,6 +857,35 @@ class Scene {
     });
   }
 
+  /**
+     * Load a full media_url via Tauri (without browser cors)
+     * @param media_url
+     * @param onComplete
+     * @returns
+     */
+  private async load_splat_wrapped_no_cors(
+    media_url: string,
+    onComplete: () => void,
+  ): Promise<SplatMesh> {
+    return new Promise(async (resolve, reject) => {
+      let buffer;
+      try {
+        buffer = await LoadWithoutCors(media_url); // Load via Tauri!
+      } catch (error) {
+        console.error("load Splat from Tauri error:", error);
+        reject(error);
+        return;
+      }
+
+      new SplatMesh({
+        fileBytes: buffer, onLoad: (mesh) => {
+          resolve(mesh);
+          onComplete();
+        }
+      });
+    });
+  }
+
   /**
    * Using a constructed media_url load a glb, grab the progress.
    * @param media_url

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/timeline.ts

@@ -190,7 +190,7 @@ export class TimeLine {
           this.queueNewObjectMessage(
             newObject,
             data.data as MediaItem,
-            AssetType.OBJECT,
+            newObject.type as AssetType
           );
         const result = this.editorEngine.sceneManager?.render_outliner(
           this.characters,
@@ -321,7 +321,7 @@ export class TimeLine {
   queueNewObjectMessage(
     item: THREE.Object3D<THREE.Object3DEventMap>,
     data: MediaItem,
-    asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE,
+    asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE | AssetType.SPLAT,
   ) {
     Queue.publish({
       queueName: QueueNames.FROM_ENGINE,

frontend/apps/editor3d/app/src/pages/PageEnigma/enums/MediaFileType.ts

@@ -16,6 +16,9 @@ export enum MediaFileType {
   // open source 3D file format.
   GLB = "glb",
 
+  // SPLAT file type. SPZ format is a compressed and smaller format.
+  SPZ = "spz",
+
   // MMD is the format for anime characters.
   // open source 3D file format.
   MMD = "mmd",

frontend/libs/components/viewer-3d/src/lib/viewer-3d.tsx

@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react";
 import * as THREE from "three";
 import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
 import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
+import { SplatMesh } from "@sparkjsdev/spark";
 
 export interface Viewer3DProps {
   modelUrl?: string;
@@ -27,7 +28,7 @@ export function Viewer3D({
   const controlsRef = useRef<OrbitControls | null>(null);
   const animationFrameRef = useRef<number | null>(null);
   const cubeRef = useRef<THREE.Mesh | null>(null);
-  const loadedModelRef = useRef<THREE.Group | null>(null);
+  const loadedModelRef = useRef<THREE.Object3D | null>(null);
   const gridRef = useRef<THREE.GridHelper | null>(null);
   const [isModelLoaded, setIsModelLoaded] = useState(false);
   const thumbnailCapturedRef = useRef(false);
@@ -191,7 +192,6 @@ export function Viewer3D({
     thumbnailCapturedRef.current = false;
 
     const scene = sceneRef.current;
-    const loader = new GLTFLoader();
 
     if (loadedModelRef.current) {
       console.log("[Viewer3D] Removing previous model");
@@ -203,95 +203,112 @@ export function Viewer3D({
       cubeRef.current.visible = false;
     }
 
-    loader.load(
-      modelUrl,
-      (gltf) => {
-        console.log("[Viewer3D] Model loaded successfully");
-        const model = gltf.scene;
+    const onModelLoaded = (model: THREE.Object3D) => {
+      const box = new THREE.Box3().setFromObject(model);
+      const size = box.getSize(new THREE.Vector3());
 
-        const box = new THREE.Box3().setFromObject(model);
-        const size = box.getSize(new THREE.Vector3());
+      const maxDim = Math.max(size.x, size.y, size.z);
+      const scale = 2 / maxDim;
+      model.scale.multiplyScalar(scale);
 
-        const maxDim = Math.max(size.x, size.y, size.z);
-        const scale = 2 / maxDim;
-        model.scale.multiplyScalar(scale);
+      const scaledBox = new THREE.Box3().setFromObject(model);
+      const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
+      const scaledSize = scaledBox.getSize(new THREE.Vector3());
 
-        const scaledBox = new THREE.Box3().setFromObject(model);
-        const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
-        const scaledSize = scaledBox.getSize(new THREE.Vector3());
+      model.position.x = -scaledCenter.x;
+      model.position.z = -scaledCenter.z;
+      model.position.y = -scaledBox.min.y;
 
-        model.position.x = -scaledCenter.x;
-        model.position.z = -scaledCenter.z;
-        model.position.y = -scaledBox.min.y;
+      scene.add(model);
+      loadedModelRef.current = model;
+      setIsModelLoaded(true);
 
-        scene.add(model);
-        loadedModelRef.current = model;
-        setIsModelLoaded(true);
+      const modelHeight = scaledSize.y;
+      const maxModelDim = Math.max(scaledSize.x, scaledSize.y, scaledSize.z);
 
-        const modelHeight = scaledSize.y;
-        const maxModelDim = Math.max(scaledSize.x, scaledSize.y, scaledSize.z);
+      // Calculate distance needed to fit the model based on camera FOV (50 degrees)
+      const fov = 50;
+      const fovRad = (fov * Math.PI) / 180;
+      const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2);
 
-        // Calculate distance needed to fit the model based on camera FOV (50 degrees)
-        const fov = 50;
-        const fovRad = (fov * Math.PI) / 180;
-        const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2);
+      // Add padding (1.4x) to ensure model isn't touching edges
+      const cameraDistance = fitDistance * 1.4;
 
-        // Add padding (1.4x) to ensure model isn't touching edges
-        const cameraDistance = fitDistance * 1.4;
-
-        if (cameraRef.current && controlsRef.current) {
-          // Position camera at 45-degree angle
-          const angle = Math.PI / 4;
-          cameraRef.current.position.set(
-            Math.sin(angle) * cameraDistance,
-            modelHeight * 0.5 + cameraDistance * 0.35,
-            Math.cos(angle) * cameraDistance
-          );
-          controlsRef.current.target.set(0, modelHeight * 0.4, 0);
-          controlsRef.current.update();
-        }
-
-        // Capture thumbnail after a short delay to ensure rendering is complete
-        if (onThumbnailCapture && !thumbnailCapturedRef.current) {
-          thumbnailCapturedRef.current = true;
-          setTimeout(() => {
-            if (rendererRef.current && sceneRef.current && cameraRef.current) {
-              // Temporarily hide grid for thumbnail
-              if (gridRef.current) {
-                gridRef.current.visible = false;
-              }
-              // Also hide placeholder cube
-              if (cubeRef.current) {
-                cubeRef.current.visible = false;
-              }
-
-              // Render and capture
-              rendererRef.current.render(sceneRef.current, cameraRef.current);
-              const dataUrl =
-                rendererRef.current.domElement.toDataURL("image/png");
-              onThumbnailCapture(dataUrl);
-
-              // Restore grid visibility
-              if (gridRef.current) {
-                gridRef.current.visible = showGrid;
-              }
-            }
-          }, 100);
-        }
-      },
-      (progress) => {
-        console.log(
-          "[Viewer3D] Loading progress:",
-          ((progress.loaded / progress.total) * 100).toFixed(2) + "%"
+      if (cameraRef.current && controlsRef.current) {
+        // Position camera at 45-degree angle
+        const angle = Math.PI / 4;
+        cameraRef.current.position.set(
+          Math.sin(angle) * cameraDistance,
+          modelHeight * 0.5 + cameraDistance * 0.35,
+          Math.cos(angle) * cameraDistance
         );
-      },
-      (error) => {
-        console.error("[Viewer3D] Error loading model:", error);
-        if (cubeRef.current) {
-          cubeRef.current.visible = true;
-        }
+        controlsRef.current.target.set(0, modelHeight * 0.4, 0);
+        controlsRef.current.update();
       }
-    );
+
+      // Capture thumbnail after a short delay to ensure rendering is complete
+      if (onThumbnailCapture && !thumbnailCapturedRef.current) {
+        thumbnailCapturedRef.current = true;
+        setTimeout(() => {
+          if (rendererRef.current && sceneRef.current && cameraRef.current) {
+            // Temporarily hide grid for thumbnail
+            if (gridRef.current) {
+              gridRef.current.visible = false;
+            }
+            // Also hide placeholder cube
+            if (cubeRef.current) {
+              cubeRef.current.visible = false;
+            }
+
+            // Render and capture
+            rendererRef.current.render(sceneRef.current, cameraRef.current);
+            const dataUrl =
+              rendererRef.current.domElement.toDataURL("image/png");
+            onThumbnailCapture(dataUrl);
+
+            // Restore grid visibility
+            if (gridRef.current) {
+              gridRef.current.visible = showGrid;
+            }
+          }
+        }, 100);
+      }
+    }
+
+    if (modelUrl.endsWith(".spz")) {
+      console.log("[Viewer3D] .spz format detected");
+      new SplatMesh({
+        url: modelUrl, onLoad: (mesh) => {
+          scene.add(mesh);
+          setIsModelLoaded(true);
+          // onModelLoaded(mesh);
+          console.log("Splat loaded")
+        }
+      });
+    } else {
+
+      const loader = new GLTFLoader();
+      loader.load(
+        modelUrl,
+        (gltf) => {
+          console.log("[Viewer3D] Model loaded successfully");
+          const model = gltf.scene;
+          onModelLoaded(model);
+        },
+        (progress) => {
+          console.log(
+            "[Viewer3D] Loading progress:",
+            ((progress.loaded / progress.total) * 100).toFixed(2) + "%"
+          );
+        },
+        (error) => {
+          console.error("[Viewer3D] Error loading model:", error);
+          if (cubeRef.current) {
+            cubeRef.current.visible = true;
+          }
+        }
+      );
+    }
 
     return () => {
       if (loadedModelRef.current && sceneRef.current) {