server/artcraft
1
0
Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-10-20 17:02:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

view meshes and splats in 3d modal (#1778)

Browse Source
This commit is contained in:
Brandon Thomas
2026-07-12 04:34:36 -04:00
committed by GitHub
parent e0047d9410
commit 9898c6dbcf
7 changed files with 191 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/apps/artcraft-webapp/vite.config.ts
+67 -17
View File
@@ -7,6 +7,70 @@ import { nxViteTsPaths } from '@nx/vite/plugins/nx-tsconfig-paths.plugin';
import { execSync } from 'child_process';
import path from 'path';
// Dev-only CDN proxy. Media assets (splats, GLBs, images) are fetched with
// fetch(), but the CDNs only send CORS headers for the production origin —
// not localhost. The web adapter's fetchAsset rewrites absolute CDN URLs to
// `/__cdn/<host>/<path>` in DEV; this middleware forwards them server-side
// (no browser CORS) to whatever host is embedded in the path.
//
// NB: This must be a middleware, not a `server.proxy` entry. The previous
// proxy config used a `router` option to pick the target host from the path,
// but `router` is an http-proxy-middleware feature — Vite's http-proxy
// silently ignores it, so every request went to the hardcoded default target
// and any asset on a different host (e.g. the dev R2 bucket) came back 403.
function cdnProxyPlugin() {
return {
name: 'cdn-proxy',
configureServer(server: {
middlewares: {
use: (
route: string,
handler: (
req: import('http').IncomingMessage,
res: import('http').ServerResponse,
) => void,
) => void;
};
}) {
server.middlewares.use('/__cdn', async (req, res) => {
try {
// Connect strips the `/__cdn` mount prefix from req.url.
const match = (req.url ?? '').match(/^\/([^/?]+)(\/[^?]*)?(\?.*)?$/);
if (!match) {
res.statusCode = 400;
res.end('Bad /__cdn path; expected /__cdn/<host>/<path>');
return;
}
const [, host, assetPath = '/', query = ''] = match;
const upstream = await fetch(`https://${host}${assetPath}${query}`, {
method: req.method === 'HEAD' ? 'HEAD' : 'GET',
headers: req.headers.range
? { range: String(req.headers.range) }
: undefined,
});
res.statusCode = upstream.status;
for (const header of [
'content-type',
'content-length',
'accept-ranges',
'content-range',
'etag',
'cache-control',
]) {
const value = upstream.headers.get(header);
if (value) res.setHeader(header, value);
}
const body = Buffer.from(await upstream.arrayBuffer());
res.end(body);
} catch (e) {
res.statusCode = 502;
res.end(`CDN proxy error: ${e}`);
}
});
},
};
}
// Custom plugin to generate news.json on dev server start
function generateNewsPlugin() {
return {
@@ -38,29 +102,15 @@ export default defineConfig(() => ({
Origin: 'https://api.storyteller.ai',
},
},
// Dev-only CDN proxy. Media assets (splats, GLBs, images) are fetched
// from the CDN with fetch(), but the CDN only sends CORS headers for the
// production origin — not localhost. The web adapter's fetchAsset rewrites
// absolute CDN URLs to `/__cdn/<host>/<path>` in DEV; this forwards them
// server-side (no browser CORS). The target host is taken from the path
// so any cdn-N.fakeyou.com / Cloudflare host works.
'/__cdn': {
target: 'https://cdn-2.fakeyou.com',
changeOrigin: true,
secure: true,
router: (req) => {
const match = req.url?.match(/^\/__cdn\/([^/]+)/);
return match ? `https://${match[1]}` : 'https://cdn-2.fakeyou.com';
},
rewrite: (path) => path.replace(/^\/__cdn\/[^/]+/, ''),
},
// NB: `/__cdn` asset proxying lives in `cdnProxyPlugin` above — it
// needs per-request host routing, which `server.proxy` can't do.
},
},
preview:{
port: 4301,
host: 'localhost',
},
plugins: [generateNewsPlugin(), nxViteTsPaths(), react(), wasm(), topLevelAwait()],
plugins: [cdnProxyPlugin(), generateNewsPlugin(), nxViteTsPaths(), react(), wasm(), topLevelAwait()],
// Uncomment this if you are using workers.
// worker: {
// plugins: [ nxViteTsPaths() ],
frontend/libs/common/src/lib/utils/thumbnail-utils.ts
+1 -1
View File
@@ -145,7 +145,7 @@ export function getPlaceholderForMediaClass(mediaClass?: string): string {
}
// File extensions for actual 3D-model assets (meshes + gaussian splats).
export const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz"];
export const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz", ".obj", ".ply"];
/** Whether a URL points at a real 3D-model asset (vs. e.g. a cover screenshot).
* Used to keep 3D-model cover images — which the backend can surface as
frontend/libs/components/gallery-modal/src/lib/GalleryDraggableItem.tsx
+3 -1
View File
@@ -426,7 +426,9 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
<span>
<span className="font-bold">Drag</span>
<span className="opacity-50">
{item.mediaClass === "dimensional"
{item.mediaClass === "dimensional" ||
item.mediaClass === "mesh" ||
item.mediaClass === "splat"
? " to add to scene"
: " to add"}
</span>
frontend/libs/components/gallery-modal/src/lib/GalleryItemMenuItems.tsx
+3 -1
View File
@@ -73,7 +73,9 @@ export const GalleryItemMenuItems: React.FC<GalleryItemMenuItemsProps> = ({
<span>
{item.mediaClass === "video"
? "View video"
: item.mediaClass === "dimensional"
: item.mediaClass === "dimensional" ||
item.mediaClass === "mesh" ||
item.mediaClass === "splat"
? "View 3D"
: "View image"}
</span>
frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx
+93 -11
View File
@@ -60,6 +60,7 @@ import {
faImage,
faVideo,
faCube,
faGlobe,
faUpload,
faExpand,
faCompress,
@@ -88,6 +89,8 @@ interface GalleryCacheEntry {
pageIndex: number;
hasMore: boolean;
timestamp: number;
/** Keyset cursor for the mesh/splat tabs (their endpoints paginate by cursor). */
cursor?: string;
}
const galleryCacheMap = new Map<string, GalleryCacheEntry>();
@@ -304,29 +307,37 @@ const SIDEBAR_FILTERS = [
{ id: "all", label: "All Assets", icon: faBorderAll },
{ id: "image", label: "Image", icon: faImage },
{ id: "video", label: "Video", icon: faVideo },
{ id: "3d", label: "3D", icon: faCube },
{ id: "mesh", label: "Mesh", icon: faCube },
{ id: "splat", label: "Splat", icon: faGlobe },
{ id: "uploaded", label: "Uploaded", icon: faUpload },
];
// Media classes for the user-list endpoint. The Mesh and Splat tabs don't use
// it — they call the session mesh/splat list endpoints instead.
const getFilterMediaClass = (filter: string) => {
switch (filter) {
case "image":
return [FilterMediaClasses.IMAGE];
case "video":
return [FilterMediaClasses.VIDEO];
case "3d":
return [FilterMediaClasses.DIMENSIONAL];
case "uploaded":
return [
FilterMediaClasses.IMAGE,
FilterMediaClasses.VIDEO,
FilterMediaClasses.DIMENSIONAL,
FilterMediaClasses.MESH,
FilterMediaClasses.SPLAT,
];
default:
return undefined;
}
};
// The backend used to file all 3D media under the (now deprecated) coarse
// "dimensional" class; new records are "mesh" or "splat". Treat all three as 3D.
const is3DClass = (mediaClass: string | undefined | null): boolean =>
mediaClass === "dimensional" || mediaClass === "mesh" || mediaClass === "splat";
const getLabel = (item: any) => {
if (!!item.maybe_title) {
return item.maybe_title;
@@ -337,7 +348,10 @@ const getLabel = (item: any) => {
case "video":
return "Video Generation";
case "dimensional":
case "mesh":
return "3D Object Generation";
case "splat":
return "3D World Generation";
default:
return "Generation";
}
@@ -791,6 +805,10 @@ export const GalleryModal = React.memo(
[],
);
// Keyset cursors for the mesh/splat tabs (their endpoints paginate by
// cursor rather than page index), keyed by filter id.
const byClassCursorRef = useRef<Record<string, string | undefined>>({});
// Map raw API item to GalleryItem
const mapApiItem = useCallback(
(item: any): GalleryItem => ({
@@ -799,7 +817,7 @@ export const GalleryModal = React.memo(
thumbnail:
item.media_class === "video"
? item.media_links.maybe_video_previews?.animated
: item.media_class === "dimensional"
: is3DClass(item.media_class)
? // Prefer the modern CDN link; fall back to the deprecated
// bucket URL for older responses.
(item.cover_image?.maybe_links?.cdn_url ??
@@ -815,13 +833,15 @@ export const GalleryModal = React.memo(
(item.filter_media_classes ? item.filter_media_classes[0] : "image"),
isUpload: item.origin_category === "upload",
assetType:
item.media_class === "dimensional"
item.media_class === "dimensional" || item.media_class === "mesh"
? item.maybe_animation_type ||
item.origin_product_category === "character" ||
(item.origin && item.origin.product_category === "character")
? "character"
: "object"
: undefined,
: item.media_class === "splat"
? "splat"
: undefined,
}),
[],
);
@@ -834,7 +854,7 @@ export const GalleryModal = React.memo(
id: item.token,
label: getLabel(item),
thumbnail:
item.media_class === "dimensional"
is3DClass(item.media_class)
? (item.cover_image?.maybe_links?.cdn_url ??
item.cover_image?.maybe_cover_image_public_bucket_url ??
null)
@@ -864,6 +884,63 @@ export const GalleryModal = React.memo(
setLoading(true);
if (!reset) setPaginationLoading(true);
try {
if (currentFilter === "mesh" || currentFilter === "splat") {
// The Mesh / Splat tabs use the session by-class list endpoints
// (server-scoped, cursor-paginated) instead of the user list.
const cursor = reset
? undefined
: byClassCursorRef.current[currentFilter];
const response =
currentFilter === "splat"
? await mediaFilesApi.ListSessionSplatMediaFiles({
cursor,
page_size: PAGE_SIZE,
})
: await mediaFilesApi.ListSessionMeshMediaFiles({
cursor,
page_size: PAGE_SIZE,
});
if (response.success && response.data) {
setItemsLoadError(null);
const newItems = response.data.map(mapApiItem);
if (reset) {
setAllItems(newItems);
} else {
setAllItems((prev) => [...prev, ...newItems]);
}
const nextCursor = response.pagination?.maybe_next ?? undefined;
byClassCursorRef.current[currentFilter] = nextCursor;
const more = newItems.length >= PAGE_SIZE && !!nextCursor;
setPageIndex(0);
setHasMore(more);
const cacheKey = getCacheKey();
setAllItems((latest) => {
galleryCacheMap.set(cacheKey, {
items: latest,
pageIndex: 0,
hasMore: more,
timestamp: Date.now(),
cursor: nextCursor,
});
return latest;
});
} else {
setItemsLoadError(
response.errorMessage || "Request failed (unknown error)",
);
}
setLoading(false);
setPaginationLoading(false);
setInitialLoading(false);
isLoadingRef.current = false;
return;
}
const filterMediaClasses = getFilterMediaClass(currentFilter);
const query = {
filter_media_classes: filterMediaClasses,
@@ -871,7 +948,6 @@ export const GalleryModal = React.memo(
include_user_uploads:
currentFilter === "uploaded" ||
currentFilter === "all" ||
currentFilter === "3d" ||
currentFilter === "image" ||
currentFilter === "video",
user_uploads_only: currentFilter === "uploaded",
@@ -970,7 +1046,7 @@ export const GalleryModal = React.memo(
setInitialLoading(false);
isLoadingRef.current = false;
},
[api, mapApiItem, getCacheKey],
[api, mediaFilesApi, mapApiItem, getCacheKey],
);
// refresh logic — shows cached items immediately, then background-refreshes
@@ -985,6 +1061,7 @@ export const GalleryModal = React.memo(
setAllItems(cached.items);
setPageIndex(cached.pageIndex);
setHasMore(cached.hasMore);
byClassCursorRef.current[activeFilterRef.current] = cached.cursor;
setInitialLoading(false);
// Background refresh — silently fetch page 0 and merge new items
loadItems(true);
@@ -2004,7 +2081,8 @@ export const GalleryModal = React.memo(
if ((item as any).mediaType === "scene_json") return false;
if (!showUploads && activeFilter !== "uploaded" && item.isUpload)
return false;
if (activeFilter === "3d") return item.mediaClass === "dimensional";
if (activeFilter === "mesh") return item.mediaClass === "mesh";
if (activeFilter === "splat") return item.mediaClass === "splat";
if (activeFilter === "image") return item.mediaClass === "image";
if (activeFilter === "video") return item.mediaClass === "video";
if (activeFilter === "all") {
@@ -2341,10 +2419,14 @@ export const GalleryModal = React.memo(
? maxSelections === 1
? "Select Video"
: "Select Videos"
: activeFilter === "3d"
: activeFilter === "mesh"
? maxSelections === 1
? "Select 3D Object"
: "Select 3D Objects"
: activeFilter === "splat"
? maxSelections === 1
? "Select 3D World"
: "Select 3D Worlds"
: activeFilter === "uploaded"
? maxSelections === 1
? "Select Upload"
frontend/libs/components/gallery-modal/src/lib/galleryDnd.ts
+3 -1
View File
@@ -284,7 +284,9 @@ function onPointerUp(event: PointerEvent) {
} else if (
dragState.modalHidden &&
(dragState.item.mediaClass === "image" ||
dragState.item.mediaClass === "dimensional")
dragState.item.mediaClass === "dimensional" ||
dragState.item.mediaClass === "mesh" ||
dragState.item.mediaClass === "splat")
) {
// The cursor left the modal and dropped onto the canvas (not a folder) —
// add to the scene. Gated on `modalHidden` so a missed folder drop *inside*
frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx
+21 -3
View File
@@ -404,17 +404,35 @@ export const Stage3DBody = ({
position.y,
);
try {
if (item.mediaClass === "dimensional") {
if (
item.mediaClass === "dimensional" ||
item.mediaClass === "mesh" ||
item.mediaClass === "splat"
) {
const isCharacter = item.assetType === "character";
const isSplat = item.mediaClass === "splat";
const mediaItem: MediaItem = {
version: 1,
type: isCharacter ? AssetType.CHARACTER : AssetType.OBJECT,
type: isCharacter
? AssetType.CHARACTER
: isSplat
? AssetType.SPLAT
: AssetType.OBJECT,
media_id: item.id || uuidv4(),
name: item.label || (isCharacter ? "Character" : "3D Object"),
name:
item.label ||
(isCharacter
? "Character"
: isSplat
? "3D World"
: "3D Object"),
};
if (isCharacter) {
await addCharacter(editor, mediaItem, worldPosition);
} else {
// NB: Splats intentionally route through addObject too — the
// engine's sceneManager resolves the media token to a splat
// (same as the engine's own DnD in DndAsset.ts).
await addObject(editor, mediaItem, worldPosition);
}
} else {