mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-29 04:52:51 -04:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
634c56a642 | ||
|
|
5e2f82faeb | ||
|
|
a8e375ff64 | ||
|
|
e5f6d87f39 | ||
|
|
8bb8c186a3 | ||
|
|
853f1d2b69 |
@@ -1,3 +1,5 @@
|
||||
import type { Model3DParams } from "./utilities/render3DModel";
|
||||
|
||||
export type NodeType = 'rectangle' | 'circle' | 'triangle' | 'image';
|
||||
|
||||
export interface NodeProps {
|
||||
@@ -22,6 +24,10 @@ export interface NodeProps {
|
||||
offsetY?: number;
|
||||
zIndex?: number;
|
||||
locked?: boolean;
|
||||
/** If set, this image node was rendered from a 3D model and can be re-edited in 3D. */
|
||||
modelUrl?: string;
|
||||
/** Last-used Three.js camera/fov/scale params for re-editing the 3D model. */
|
||||
model3dParams?: Model3DParams;
|
||||
}
|
||||
|
||||
export class Node {
|
||||
@@ -46,7 +52,9 @@ export class Node {
|
||||
offsetY: number;
|
||||
zIndex: number;
|
||||
locked: boolean;
|
||||
|
||||
modelUrl?: string;
|
||||
model3dParams?: Model3DParams;
|
||||
|
||||
constructor({
|
||||
id,
|
||||
x,
|
||||
@@ -68,7 +76,9 @@ export class Node {
|
||||
offsetX = 0,
|
||||
offsetY = 0,
|
||||
zIndex = 0,
|
||||
locked = false
|
||||
locked = false,
|
||||
modelUrl,
|
||||
model3dParams,
|
||||
}: NodeProps) {
|
||||
this.id = id;
|
||||
this.x = x;
|
||||
@@ -91,6 +101,8 @@ export class Node {
|
||||
this.offsetY = offsetY;
|
||||
this.zIndex = zIndex;
|
||||
this.locked = locked;
|
||||
this.modelUrl = modelUrl;
|
||||
this.model3dParams = model3dParams;
|
||||
}
|
||||
|
||||
setPosition(x: number, y: number): void {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState, useRef, useEffect, useCallback, useMemo } from "react";
|
||||
import React, { useState, useRef, useEffect, useCallback, useMemo, memo } from "react";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
import { DRAW_LAYER_ID, INPAINT_LAYER_ID, PaintSurface } from "./PaintSurface";
|
||||
import "./App.css";
|
||||
@@ -45,9 +45,209 @@ import { EncodeImageBitmapToBase64 } from "./utilities/EncodeImageBitmapToBase64
|
||||
import { RefImage, usePrompt2DStore } from "@storyteller/ui-promptbox";
|
||||
import { PromptsApi } from "@storyteller/api";
|
||||
import toast from "react-hot-toast";
|
||||
import {
|
||||
render3DModelToDataUrl,
|
||||
DEFAULT_MODEL3D_PARAMS,
|
||||
type Model3DParams,
|
||||
} from "./utilities/render3DModel";
|
||||
import {
|
||||
Model3DOverlay,
|
||||
type Model3DOverlayHandle,
|
||||
} from "./components/Model3DOverlay";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faUpRightAndDownLeftFromCenter,
|
||||
faArrowsRotate,
|
||||
faCamera,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
|
||||
const PAGE_ID: ModelPage = ModelPage.Canvas2D;
|
||||
|
||||
// ─── Edit3DButton ─────────────────────────────────────────────────────────────
|
||||
// Isolated memoized component so Konva-driven position updates (stage pan/zoom,
|
||||
// transformer drag) only re-render this small button — not the entire PageDraw
|
||||
// tree. It owns the forceUpdate state and Konva event listeners internally.
|
||||
interface Edit3DButtonProps {
|
||||
nodeId: string;
|
||||
stageRef: { current: Konva.Stage };
|
||||
onEdit: (nodeId: string) => void;
|
||||
}
|
||||
|
||||
const Edit3DButton = memo(function Edit3DButton({
|
||||
nodeId,
|
||||
stageRef,
|
||||
onEdit,
|
||||
}: Edit3DButtonProps) {
|
||||
const [, forceUpdate] = useState(0);
|
||||
const bump = useCallback(() => forceUpdate((t) => t + 1), []);
|
||||
const [interacting, setInteracting] = useState(false);
|
||||
|
||||
// Attach Konva event listeners:
|
||||
// - Stage pan/zoom → bump() to reposition
|
||||
// - Transformer start/end → hide/show button to avoid jitter
|
||||
// - Node drag start/end → hide/show button
|
||||
useEffect(() => {
|
||||
const stage = stageRef.current;
|
||||
if (!stage?.on) return;
|
||||
|
||||
const ns = ".edit3dbtn";
|
||||
stage.on(
|
||||
`dragmove${ns} xChange${ns} yChange${ns} scaleXChange${ns} scaleYChange${ns}`,
|
||||
bump,
|
||||
);
|
||||
|
||||
const transformers = stage.find("Transformer");
|
||||
transformers.forEach((tr) => {
|
||||
tr.on(`transformstart${ns}`, () => setInteracting(true));
|
||||
// Defer one rAF so Konva finishes writing the node's new attributes
|
||||
// (scaleX/Y, width/height) before we read getClientRect() in the render.
|
||||
tr.on(`transformend${ns}`, () => requestAnimationFrame(() => { setInteracting(false); bump(); }));
|
||||
});
|
||||
|
||||
const konvaNode = stage.findOne("#" + nodeId);
|
||||
if (konvaNode) {
|
||||
konvaNode.on(`dragstart${ns}`, () => setInteracting(true));
|
||||
konvaNode.on(`dragend${ns}`, () => requestAnimationFrame(() => { setInteracting(false); bump(); }));
|
||||
}
|
||||
|
||||
return () => {
|
||||
stage.off(ns);
|
||||
transformers.forEach((tr) => tr.off(ns));
|
||||
konvaNode?.off(ns);
|
||||
};
|
||||
}, [stageRef, bump, nodeId]);
|
||||
|
||||
if (interacting) return null;
|
||||
|
||||
const stage = stageRef.current;
|
||||
if (!stage?.container) return null;
|
||||
|
||||
const konvaNode = stage.findOne("#" + nodeId) as Konva.Shape | undefined;
|
||||
if (!konvaNode) return null;
|
||||
|
||||
// getClientRect() returns the AABB in stage-container coords after all
|
||||
// transforms (rotation, scale, stage pan/zoom) — no manual math needed.
|
||||
const clientRect = konvaNode.getClientRect();
|
||||
const stageContainerRect = stage.container().getBoundingClientRect();
|
||||
const btnLeft = stageContainerRect.left + clientRect.x + clientRect.width / 2;
|
||||
const btnTop = stageContainerRect.top + clientRect.y + clientRect.height / 2;
|
||||
|
||||
return (
|
||||
<button
|
||||
className="pointer-events-auto fixed z-40 -translate-x-1/2 -translate-y-1/2 rounded-full bg-blue-600 px-3 py-1 text-xs font-semibold text-white shadow-lg hover:bg-blue-500"
|
||||
style={{ left: btnLeft, top: btnTop }}
|
||||
onPointerDown={(e) => e.stopPropagation()}
|
||||
onClick={() => onEdit(nodeId)}
|
||||
>
|
||||
Edit 3D
|
||||
</button>
|
||||
);
|
||||
});
|
||||
|
||||
// ─── DragScrubButton ──────────────────────────────────────────────────────────
|
||||
// Round button that scrubs a 3D parameter on horizontal+vertical drag.
|
||||
// Uses pointer capture so the drag continues outside the button bounds.
|
||||
function DragScrubButton({
|
||||
icon,
|
||||
title,
|
||||
onDrag,
|
||||
}: {
|
||||
icon: React.ReactNode;
|
||||
title: string;
|
||||
onDrag: (dx: number, dy: number) => void;
|
||||
}) {
|
||||
const onPointerDown = (e: React.PointerEvent<HTMLButtonElement>) => {
|
||||
e.stopPropagation();
|
||||
e.currentTarget.setPointerCapture(e.pointerId);
|
||||
};
|
||||
const onPointerMove = (e: React.PointerEvent<HTMLButtonElement>) => {
|
||||
if (e.buttons === 0) return;
|
||||
onDrag(e.movementX, e.movementY);
|
||||
};
|
||||
return (
|
||||
<button
|
||||
title={title}
|
||||
className="flex h-10 w-10 cursor-move items-center justify-center rounded-full bg-black/70 text-white shadow-lg hover:bg-black/90 active:bg-blue-600"
|
||||
onPointerDown={onPointerDown}
|
||||
onPointerMove={onPointerMove}
|
||||
>
|
||||
{icon}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Edit3DScrubControls ──────────────────────────────────────────────────────
|
||||
// Shown in place of Edit3DButton while the 3D overlay is open. Renders the
|
||||
// scrub buttons (scale, rotate, FOV, apply) at the same node-center position,
|
||||
// communicating with Model3DOverlay via an imperative handle ref.
|
||||
interface Edit3DScrubControlsProps {
|
||||
nodeId: string;
|
||||
stageRef: { current: Konva.Stage };
|
||||
overlayHandle: React.RefObject<Model3DOverlayHandle>;
|
||||
}
|
||||
|
||||
const Edit3DScrubControls = memo(function Edit3DScrubControls({
|
||||
nodeId,
|
||||
stageRef,
|
||||
overlayHandle,
|
||||
}: Edit3DScrubControlsProps) {
|
||||
const [, forceUpdate] = useState(0);
|
||||
const bump = useCallback(() => forceUpdate((t) => t + 1), []);
|
||||
|
||||
useEffect(() => {
|
||||
const stage = stageRef.current;
|
||||
if (!stage?.on) return;
|
||||
const ns = ".edit3dcontrols";
|
||||
stage.on(
|
||||
`dragmove${ns} xChange${ns} yChange${ns} scaleXChange${ns} scaleYChange${ns}`,
|
||||
bump,
|
||||
);
|
||||
return () => {
|
||||
stage.off(ns);
|
||||
};
|
||||
}, [stageRef, bump, nodeId]);
|
||||
|
||||
const stage = stageRef.current;
|
||||
if (!stage?.container) return null;
|
||||
const konvaNode = stage.findOne("#" + nodeId) as Konva.Shape | undefined;
|
||||
if (!konvaNode) return null;
|
||||
|
||||
const clientRect = konvaNode.getClientRect();
|
||||
const stageContainerRect = stage.container().getBoundingClientRect();
|
||||
const cx = stageContainerRect.left + clientRect.x + clientRect.width / 2;
|
||||
const cy = stageContainerRect.top + clientRect.y + clientRect.height / 2;
|
||||
|
||||
return (
|
||||
<div
|
||||
className="pointer-events-auto fixed z-[60] flex -translate-x-1/2 -translate-y-1/2 items-center gap-2"
|
||||
style={{ left: cx, top: cy }}
|
||||
>
|
||||
<DragScrubButton
|
||||
icon={<FontAwesomeIcon icon={faUpRightAndDownLeftFromCenter} />}
|
||||
title="Scale — drag"
|
||||
onDrag={(dx, dy) => overlayHandle.current?.onScaleDrag(dx, dy)}
|
||||
/>
|
||||
<DragScrubButton
|
||||
icon={<FontAwesomeIcon icon={faArrowsRotate} />}
|
||||
title="Rotate — drag"
|
||||
onDrag={(dx, dy) => overlayHandle.current?.onRotateDrag(dx, dy)}
|
||||
/>
|
||||
<DragScrubButton
|
||||
icon={<FontAwesomeIcon icon={faCamera} />}
|
||||
title="Field of view — drag"
|
||||
onDrag={(dx, dy) => overlayHandle.current?.onFovDrag(dx, dy)}
|
||||
/>
|
||||
<button
|
||||
onClick={() => overlayHandle.current?.commit()}
|
||||
className="flex h-10 w-10 items-center justify-center rounded-full bg-blue-600 text-sm font-bold text-white shadow-lg hover:bg-blue-500"
|
||||
onPointerDown={(e) => e.stopPropagation()}
|
||||
>
|
||||
✓
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
export const DecodeBase64ToImage = async (
|
||||
base64String: string,
|
||||
): Promise<ImageBitmap> => {
|
||||
@@ -77,6 +277,8 @@ const PageDraw = () => {
|
||||
const canvasWidth = useRef<number>(1024);
|
||||
const canvasHeight = useRef<number>(1024);
|
||||
const [isSelecting, setIsSelecting] = useState<boolean>(false);
|
||||
const [editing3DNodeId, setEditing3DNodeId] = useState<string | null>(null);
|
||||
const overlayHandleRef = useRef<Model3DOverlayHandle>(null);
|
||||
const stageRef = useRef<Konva.Stage>({} as Konva.Stage);
|
||||
const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({});
|
||||
|
||||
@@ -113,6 +315,8 @@ const PageDraw = () => {
|
||||
finishRemoveBackground: state.finishRemoveBackground,
|
||||
createImageFromUrl: state.createImageFromUrl,
|
||||
createImageFromFile: state.createImageFromFile,
|
||||
createImageFrom3DModel: state.createImageFrom3DModel,
|
||||
updateNode: state.updateNode,
|
||||
setBaseImageInfo: state.setBaseImageInfo,
|
||||
RESET: state.RESET,
|
||||
clearLineNodes: state.clearLineNodes,
|
||||
@@ -158,6 +362,8 @@ const PageDraw = () => {
|
||||
finishRemoveBackground,
|
||||
createImageFromUrl,
|
||||
createImageFromFile,
|
||||
createImageFrom3DModel,
|
||||
updateNode,
|
||||
setBaseImageInfo,
|
||||
RESET,
|
||||
clearLineNodes,
|
||||
@@ -274,9 +480,8 @@ const PageDraw = () => {
|
||||
|
||||
// Listen for gallery drag and drop events
|
||||
useEffect(() => {
|
||||
const handleGallery2DDrop = (event: CustomEvent) => {
|
||||
const handleGallery2DDrop = async (event: CustomEvent) => {
|
||||
const { item, canvasPosition } = event.detail;
|
||||
console.log("Received 2D gallery drop:", { item, canvasPosition });
|
||||
|
||||
const stage = stageRef.current;
|
||||
if (!stage) {
|
||||
@@ -286,18 +491,50 @@ const PageDraw = () => {
|
||||
return;
|
||||
}
|
||||
// Transform canvas coordinates to stage coordinates
|
||||
// Account for stage position, scale, and transformations
|
||||
const stageX = stage.x();
|
||||
const stageY = stage.y();
|
||||
const scaleX = stage.scaleX();
|
||||
const scaleY = stage.scaleY();
|
||||
|
||||
const stagePoint = {
|
||||
x: (canvasPosition.x - stageX) / scaleX,
|
||||
y: (canvasPosition.y - stageY) / scaleY,
|
||||
x: (canvasPosition.x - stage.x()) / stage.scaleX(),
|
||||
y: (canvasPosition.y - stage.y()) / stage.scaleY(),
|
||||
};
|
||||
|
||||
console.log("Transformed stage coordinates:", stagePoint);
|
||||
if (item.mediaClass === "dimensional") {
|
||||
const modelUrl = item.fullImage;
|
||||
if (!modelUrl) {
|
||||
console.error("No model URL available for 3D item");
|
||||
return;
|
||||
}
|
||||
const toastId = toast.loading(`Loading 3D model "${item.label}"…`);
|
||||
try {
|
||||
const dataUrl = await render3DModelToDataUrl(
|
||||
modelUrl,
|
||||
DEFAULT_MODEL3D_PARAMS,
|
||||
);
|
||||
const img = new globalThis.Image();
|
||||
img.onload = () => {
|
||||
const canvasDims = getAspectRatioDimensions();
|
||||
const maxDim = Math.min(canvasDims.width, canvasDims.height) * 0.25;
|
||||
const aspect = img.width / img.height;
|
||||
const displayW = Math.min(img.width, maxDim * Math.max(1, aspect));
|
||||
const displayH = displayW / aspect;
|
||||
createImageFrom3DModel(
|
||||
stagePoint.x - displayW / 2,
|
||||
stagePoint.y - displayH / 2,
|
||||
dataUrl,
|
||||
modelUrl,
|
||||
{ ...DEFAULT_MODEL3D_PARAMS, nativeWidth: img.width, nativeHeight: img.height },
|
||||
displayW,
|
||||
displayH,
|
||||
);
|
||||
toast.success(`Added "${item.label}" to canvas`, { id: toastId });
|
||||
};
|
||||
img.src = dataUrl;
|
||||
} catch (err) {
|
||||
console.error("Failed to render 3D model:", err);
|
||||
toast.error(`Failed to load 3D model "${item.label}"`, {
|
||||
id: toastId,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const imageUrl = item.fullImage || item.thumbnail;
|
||||
if (!imageUrl) {
|
||||
@@ -305,14 +542,7 @@ const PageDraw = () => {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("Creating image from URL:", imageUrl);
|
||||
|
||||
createImageFromUrl(stagePoint.x, stagePoint.y, imageUrl);
|
||||
|
||||
console.log(
|
||||
`Created image "${item.label}" at stage position:`,
|
||||
stagePoint,
|
||||
);
|
||||
};
|
||||
|
||||
window.addEventListener(
|
||||
@@ -326,8 +556,51 @@ const PageDraw = () => {
|
||||
handleGallery2DDrop as EventListener,
|
||||
);
|
||||
};
|
||||
// Stable action ref; no need to depend on full store.
|
||||
}, [createImageFromUrl]);
|
||||
}, [createImageFromUrl, createImageFrom3DModel]);
|
||||
|
||||
// Auto-close the 3D overlay when the editing node is no longer selected
|
||||
// (e.g. user clicks the canvas background or selects a different node).
|
||||
// We do NOT auto-open here — opening is gated behind the "Edit 3D" button
|
||||
// so the Konva transformer can work normally on selection.
|
||||
useEffect(() => {
|
||||
if (editing3DNodeId && !selectedNodeIds.includes(editing3DNodeId)) {
|
||||
setEditing3DNodeId(null);
|
||||
}
|
||||
}, [selectedNodeIds, editing3DNodeId]);
|
||||
|
||||
const handle3DOverlayCommit = useCallback(
|
||||
(dataUrl: string, params: Model3DParams) => {
|
||||
if (!editing3DNodeId) return;
|
||||
const img = new globalThis.Image();
|
||||
img.onload = () => {
|
||||
updateNode(
|
||||
editing3DNodeId,
|
||||
{ imageUrl: dataUrl, imageElement: img, model3dParams: params },
|
||||
true,
|
||||
);
|
||||
};
|
||||
img.src = dataUrl;
|
||||
setEditing3DNodeId(null);
|
||||
// Deselect so the auto-open useEffect doesn't immediately re-trigger
|
||||
// the overlay when the updated node re-appears in the Konva canvas.
|
||||
selectNode(null);
|
||||
},
|
||||
[editing3DNodeId, updateNode, selectNode],
|
||||
);
|
||||
|
||||
// Derive a display-only node list that hides the node currently open in the
|
||||
// 3D overlay. We filter here rather than in the store so the node's data
|
||||
// (position, modelUrl, model3dParams) is fully preserved and the overlay
|
||||
// can read it via `nodes.find(...)`. The Three.js canvas sits directly on
|
||||
// top of where the Konva image would be, so removing it from the render
|
||||
// list eliminates the double-image without any visual gap.
|
||||
const displayNodes = useMemo(
|
||||
() =>
|
||||
editing3DNodeId
|
||||
? nodes.filter((n) => n.id !== editing3DNodeId)
|
||||
: nodes,
|
||||
[nodes, editing3DNodeId],
|
||||
);
|
||||
|
||||
const handleImageUpload = async (files: File[]): Promise<void> => {
|
||||
// Determine current canvas dimensions from the store (according to aspect-ratio)
|
||||
@@ -940,7 +1213,7 @@ const PageDraw = () => {
|
||||
})}
|
||||
>
|
||||
<PaintSurface
|
||||
nodes={nodes}
|
||||
nodes={displayNodes}
|
||||
lineNodes={lineNodes}
|
||||
selectedNodeIds={selectedNodeIds}
|
||||
onCanvasSizeChange={(width: number, height: number): void => {
|
||||
@@ -974,6 +1247,38 @@ const PageDraw = () => {
|
||||
<CostCalculatorButton modelPage={PAGE_ID} />
|
||||
<HelpMenuButton />
|
||||
</div>
|
||||
{/* Floating "Edit 3D" button — shown above the selected 3D-model node
|
||||
when the overlay is not already open. Rendered as an isolated memoized
|
||||
component so Konva-driven position updates don't re-render PageDraw. */}
|
||||
{!editing3DNodeId &&
|
||||
selectedNodeIds.length === 1 &&
|
||||
nodes.find((n) => n.id === selectedNodeIds[0])?.modelUrl && (
|
||||
<Edit3DButton
|
||||
nodeId={selectedNodeIds[0]}
|
||||
stageRef={stageRef}
|
||||
onEdit={setEditing3DNodeId}
|
||||
/>
|
||||
)}
|
||||
{editing3DNodeId &&
|
||||
(() => {
|
||||
const editingNode = nodes.find((n) => n.id === editing3DNodeId);
|
||||
return editingNode ? (
|
||||
<>
|
||||
<Edit3DScrubControls
|
||||
nodeId={editing3DNodeId}
|
||||
stageRef={stageRef}
|
||||
overlayHandle={overlayHandleRef}
|
||||
/>
|
||||
<Model3DOverlay
|
||||
ref={overlayHandleRef}
|
||||
node={editingNode}
|
||||
stageRef={stageRef}
|
||||
onCommit={handle3DOverlayCommit}
|
||||
onDismiss={() => setEditing3DNodeId(null)}
|
||||
/>
|
||||
</>
|
||||
) : null;
|
||||
})()}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
import React, { useEffect, useRef, useCallback, useImperativeHandle } from "react";
|
||||
import * as THREE from "three";
|
||||
import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
||||
import Konva from "konva";
|
||||
import { Node } from "../Node";
|
||||
import {
|
||||
Model3DParams,
|
||||
DEFAULT_MODEL3D_PARAMS,
|
||||
} from "../utilities/render3DModel";
|
||||
|
||||
export interface Model3DOverlayHandle {
|
||||
onScaleDrag: (dx: number, dy: number) => void;
|
||||
onRotateDrag: (dx: number, dy: number) => void;
|
||||
onFovDrag: (dx: number, dy: number) => void;
|
||||
commit: () => void;
|
||||
}
|
||||
|
||||
interface Model3DOverlayProps {
|
||||
node: Node;
|
||||
stageRef: React.MutableRefObject<Konva.Stage>;
|
||||
onCommit: (imageDataUrl: string, params: Model3DParams) => void;
|
||||
onDismiss: () => void;
|
||||
}
|
||||
|
||||
/** Returns viewport-relative rect for the Konva node, suitable for `position:fixed` overlays. */
|
||||
function buildNodeScreenRect(
|
||||
node: Node,
|
||||
stageRef: React.MutableRefObject<Konva.Stage>,
|
||||
): { left: number; top: number; width: number; height: number } {
|
||||
const stage = stageRef.current;
|
||||
if (!stage) {
|
||||
return { left: 0, top: 0, width: 300, height: 300 };
|
||||
}
|
||||
|
||||
const stageContainerRect = stage.container().getBoundingClientRect();
|
||||
const stageScaleX = stage.scaleX();
|
||||
const stageScaleY = stage.scaleY();
|
||||
const nodeScaleX = node.scaleX ?? 1;
|
||||
const nodeScaleY = node.scaleY ?? 1;
|
||||
|
||||
const left = stageContainerRect.left + stage.x() + node.x * stageScaleX;
|
||||
const top = stageContainerRect.top + stage.y() + node.y * stageScaleY;
|
||||
const width = node.width * nodeScaleX * stageScaleX;
|
||||
const height = node.height * nodeScaleY * stageScaleY;
|
||||
|
||||
return { left, top, width, height };
|
||||
}
|
||||
|
||||
export const Model3DOverlay = React.forwardRef<Model3DOverlayHandle, Model3DOverlayProps>(
|
||||
function Model3DOverlay({ node, stageRef, onCommit, onDismiss }, ref) {
|
||||
const overlayRef = useRef<HTMLDivElement>(null);
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
|
||||
const sceneRef = useRef<THREE.Scene | null>(null);
|
||||
const cameraRef = useRef<THREE.PerspectiveCamera | null>(null);
|
||||
const rendererRef = useRef<THREE.WebGLRenderer | null>(null);
|
||||
const controlsRef = useRef<OrbitControls | null>(null);
|
||||
const animFrameRef = useRef<number | null>(null);
|
||||
const modelScaleRef = useRef<number>(
|
||||
node.model3dParams?.modelScale ?? DEFAULT_MODEL3D_PARAMS.modelScale,
|
||||
);
|
||||
const loadedModelRef = useRef<THREE.Object3D | null>(null);
|
||||
const committedRef = useRef(false);
|
||||
|
||||
const screenRect = buildNodeScreenRect(node, stageRef);
|
||||
|
||||
const captureParams = useCallback((): Model3DParams => {
|
||||
const cam = cameraRef.current;
|
||||
const ctrl = controlsRef.current;
|
||||
const baseParams = node.model3dParams ?? DEFAULT_MODEL3D_PARAMS;
|
||||
if (!cam || !ctrl) return baseParams;
|
||||
return {
|
||||
cameraPosition: {
|
||||
x: cam.position.x,
|
||||
y: cam.position.y,
|
||||
z: cam.position.z,
|
||||
},
|
||||
cameraTarget: {
|
||||
x: ctrl.target.x,
|
||||
y: ctrl.target.y,
|
||||
z: ctrl.target.z,
|
||||
},
|
||||
fov: cam.fov,
|
||||
modelScale: modelScaleRef.current,
|
||||
// Preserve native dims — they are fixed for the lifetime of this 3D model node
|
||||
nativeWidth: baseParams.nativeWidth,
|
||||
nativeHeight: baseParams.nativeHeight,
|
||||
};
|
||||
}, [node.model3dParams]);
|
||||
|
||||
const commit = useCallback(() => {
|
||||
if (committedRef.current) return;
|
||||
committedRef.current = true;
|
||||
|
||||
const renderer = rendererRef.current;
|
||||
const scene = sceneRef.current;
|
||||
const camera = cameraRef.current;
|
||||
|
||||
if (!renderer || !scene || !camera) {
|
||||
onDismiss();
|
||||
return;
|
||||
}
|
||||
|
||||
// Render at native (undistorted) dimensions. Konva will stretch the committed
|
||||
// PNG to fill the node's current frame, preserving any user-applied stretch.
|
||||
const baseParams = node.model3dParams ?? DEFAULT_MODEL3D_PARAMS;
|
||||
const nativeW = baseParams.nativeWidth ?? Math.round(node.width * (node.scaleX ?? 1));
|
||||
const nativeH = baseParams.nativeHeight ?? Math.round(node.height * (node.scaleY ?? 1));
|
||||
renderer.setSize(nativeW, nativeH);
|
||||
camera.aspect = nativeW / nativeH;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.render(scene, camera);
|
||||
|
||||
const dataUrl = renderer.domElement.toDataURL("image/png");
|
||||
const params = captureParams();
|
||||
onCommit(dataUrl, params);
|
||||
}, [node, captureParams, onCommit, onDismiss]);
|
||||
|
||||
const onScaleDrag = useCallback((dx: number, _dy: number) => {
|
||||
const model = loadedModelRef.current;
|
||||
if (!model) return;
|
||||
const newScale = Math.max(0.1, Math.min(3, modelScaleRef.current + dx * 0.01));
|
||||
const ratio = newScale / modelScaleRef.current;
|
||||
model.scale.multiplyScalar(ratio);
|
||||
modelScaleRef.current = newScale;
|
||||
// Reset position first so bbox is computed from origin — avoids world-space drift
|
||||
model.position.set(0, 0, 0);
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const center = box.getCenter(new THREE.Vector3());
|
||||
model.position.x = -center.x;
|
||||
model.position.z = -center.z;
|
||||
model.position.y = -box.min.y;
|
||||
}, []);
|
||||
|
||||
const onRotateDrag = useCallback((dx: number, dy: number) => {
|
||||
const cam = cameraRef.current;
|
||||
const ctrl = controlsRef.current;
|
||||
if (!cam || !ctrl) return;
|
||||
const offset = new THREE.Vector3().subVectors(cam.position, ctrl.target);
|
||||
const sph = new THREE.Spherical().setFromVector3(offset);
|
||||
sph.theta -= dx * 0.01;
|
||||
sph.phi = Math.max(0.01, Math.min(Math.PI - 0.01, sph.phi - dy * 0.01));
|
||||
offset.setFromSpherical(sph);
|
||||
cam.position.copy(ctrl.target).add(offset);
|
||||
cam.lookAt(ctrl.target);
|
||||
ctrl.update();
|
||||
}, []);
|
||||
|
||||
const onFovDrag = useCallback((dx: number, _dy: number) => {
|
||||
const cam = cameraRef.current;
|
||||
if (!cam) return;
|
||||
cam.fov = Math.max(20, Math.min(90, cam.fov + dx * 0.3));
|
||||
cam.updateProjectionMatrix();
|
||||
}, []);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
onScaleDrag,
|
||||
onRotateDrag,
|
||||
onFovDrag,
|
||||
commit,
|
||||
}), [onScaleDrag, onRotateDrag, onFovDrag, commit]);
|
||||
|
||||
// Setup Three.js scene
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
|
||||
const params = node.model3dParams ?? DEFAULT_MODEL3D_PARAMS;
|
||||
// Render Three.js at native (undistorted) dimensions. The canvas element
|
||||
// uses h-full w-full so the browser CSS-stretches the framebuffer to fill
|
||||
// the overlay div, giving a live preview of how the stretch will look.
|
||||
const nativeW = params.nativeWidth ?? Math.round(screenRect.width);
|
||||
const nativeH = params.nativeHeight ?? Math.round(screenRect.height);
|
||||
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = null;
|
||||
sceneRef.current = scene;
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({
|
||||
antialias: true,
|
||||
alpha: true,
|
||||
canvas,
|
||||
});
|
||||
renderer.setClearColor(0x000000, 0);
|
||||
// Pass false so Three.js does NOT update canvas.style.width/height — we let
|
||||
// Tailwind's h-full w-full CSS scale the native framebuffer to the overlay
|
||||
// div's dimensions, giving a live stretched/squished preview.
|
||||
renderer.setSize(nativeW, nativeH, false);
|
||||
rendererRef.current = renderer;
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(
|
||||
params.fov,
|
||||
nativeW / nativeH,
|
||||
0.1,
|
||||
1000,
|
||||
);
|
||||
camera.position.set(
|
||||
params.cameraPosition.x,
|
||||
params.cameraPosition.y,
|
||||
params.cameraPosition.z,
|
||||
);
|
||||
cameraRef.current = camera;
|
||||
|
||||
const controls = new OrbitControls(camera, canvas);
|
||||
controls.target.set(
|
||||
params.cameraTarget.x,
|
||||
params.cameraTarget.y,
|
||||
params.cameraTarget.z,
|
||||
);
|
||||
controls.enabled = false; // interaction handled via scrub buttons
|
||||
controls.update();
|
||||
controlsRef.current = controls;
|
||||
|
||||
// Lights
|
||||
scene.add(new THREE.AmbientLight(0xffffff, 2));
|
||||
const hemi = new THREE.HemisphereLight(0xffffff, 0x888888, 1.2);
|
||||
scene.add(hemi);
|
||||
const key = new THREE.DirectionalLight(0xffffff, 2);
|
||||
key.position.set(2, 10, 8);
|
||||
scene.add(key);
|
||||
const fill = new THREE.DirectionalLight(0xffffff, 1.2);
|
||||
fill.position.set(-6, 6, -4);
|
||||
scene.add(fill);
|
||||
const front = new THREE.DirectionalLight(0xffffff, 1);
|
||||
front.position.set(0, 4, 10);
|
||||
scene.add(front);
|
||||
|
||||
// Load model
|
||||
const loader = new GLTFLoader();
|
||||
loader.load(
|
||||
node.modelUrl!,
|
||||
(gltf) => {
|
||||
const model = gltf.scene;
|
||||
|
||||
// Auto-fit + apply user scale
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
const maxDim = Math.max(size.x, size.y, size.z);
|
||||
const fitScale = (2 / maxDim) * params.modelScale;
|
||||
model.scale.multiplyScalar(fitScale);
|
||||
|
||||
const scaledBox = new THREE.Box3().setFromObject(model);
|
||||
const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
|
||||
model.position.x = -scaledCenter.x;
|
||||
model.position.z = -scaledCenter.z;
|
||||
model.position.y = -scaledBox.min.y;
|
||||
|
||||
scene.add(model);
|
||||
loadedModelRef.current = model;
|
||||
},
|
||||
undefined,
|
||||
(err) => console.error("[Model3DOverlay] Failed to load model:", err),
|
||||
);
|
||||
|
||||
// RAF loop
|
||||
const animate = () => {
|
||||
animFrameRef.current = requestAnimationFrame(animate);
|
||||
renderer.render(scene, camera);
|
||||
};
|
||||
animate();
|
||||
|
||||
return () => {
|
||||
if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current);
|
||||
controls.dispose();
|
||||
scene.traverse((obj) => {
|
||||
if (obj instanceof THREE.Mesh) {
|
||||
obj.geometry?.dispose();
|
||||
const mats = Array.isArray(obj.material)
|
||||
? obj.material
|
||||
: [obj.material];
|
||||
mats.forEach((m) => m?.dispose());
|
||||
}
|
||||
});
|
||||
renderer.dispose();
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [node.modelUrl]);
|
||||
|
||||
// Keyboard handler
|
||||
useEffect(() => {
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
if (e.key === "Escape") {
|
||||
commit();
|
||||
}
|
||||
};
|
||||
window.addEventListener("keydown", onKey);
|
||||
return () => window.removeEventListener("keydown", onKey);
|
||||
}, [commit]);
|
||||
|
||||
// Click-outside to commit
|
||||
const onOverlayPointerDown = (e: React.PointerEvent) => {
|
||||
if (
|
||||
overlayRef.current &&
|
||||
!overlayRef.current.contains(e.target as Element)
|
||||
) {
|
||||
commit();
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
// Full-viewport capture layer for click-outside detection
|
||||
<div
|
||||
className="pointer-events-auto fixed inset-0 z-50"
|
||||
onPointerDown={onOverlayPointerDown}
|
||||
>
|
||||
{/* Overlay card positioned over the Konva node, CSS-rotated to match */}
|
||||
<div
|
||||
ref={overlayRef}
|
||||
className="absolute overflow-hidden rounded-lg shadow-2xl"
|
||||
style={{
|
||||
left: screenRect.left,
|
||||
top: screenRect.top,
|
||||
width: screenRect.width,
|
||||
height: screenRect.height,
|
||||
transform: `rotate(${node.rotation ?? 0}deg)`,
|
||||
transformOrigin: "0 0",
|
||||
}}
|
||||
onPointerDown={(e) => e.stopPropagation()}
|
||||
>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
className="h-full w-full"
|
||||
style={{ display: "block" }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -3,6 +3,7 @@ import { Node, NodeType } from "../Node";
|
||||
import { EnqueueImageBgRemoval } from "libs/tauri-api/src/lib/enqueue/EnqueueImageBgRemovalCommand";
|
||||
import { ImageBundle } from "~/pages/PageEdit/HistoryStack";
|
||||
import { BaseSelectorImage } from "~/pages/PageEdit/BaseImageSelector";
|
||||
import { Model3DParams } from "../utilities/render3DModel";
|
||||
|
||||
// Add LineNode type
|
||||
export type LineNode = {
|
||||
@@ -84,6 +85,8 @@ type SerializedNodeData = {
|
||||
offsetY: number;
|
||||
zIndex: number;
|
||||
locked: boolean;
|
||||
modelUrl?: string;
|
||||
model3dParams?: Model3DParams;
|
||||
};
|
||||
|
||||
interface HistoryNodeData {
|
||||
@@ -204,6 +207,17 @@ export interface SceneState {
|
||||
height?: number,
|
||||
) => void;
|
||||
|
||||
// Create an image node backed by a 3D model (dataUrl is the initial render)
|
||||
createImageFrom3DModel: (
|
||||
x: number,
|
||||
y: number,
|
||||
renderedDataUrl: string,
|
||||
modelUrl: string,
|
||||
params: Model3DParams,
|
||||
width?: number,
|
||||
height?: number,
|
||||
) => void;
|
||||
|
||||
// Action for deleting selected items
|
||||
deleteSelectedItems: () => void;
|
||||
RESET: () => void;
|
||||
@@ -557,6 +571,8 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
|
||||
offsetY: node.offsetY || 0, // Include offset in history
|
||||
zIndex: node.zIndex || 0,
|
||||
locked: node.locked || false,
|
||||
modelUrl: node.modelUrl,
|
||||
model3dParams: node.model3dParams,
|
||||
})),
|
||||
lineNodes: JSON.parse(JSON.stringify(state.lineNodes)),
|
||||
};
|
||||
@@ -810,6 +826,40 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
|
||||
}
|
||||
},
|
||||
|
||||
// Create an image node backed by a 3D model
|
||||
createImageFrom3DModel: (
|
||||
x: number,
|
||||
y: number,
|
||||
renderedDataUrl: string,
|
||||
modelUrl: string,
|
||||
params: Model3DParams,
|
||||
width = 512,
|
||||
height = 512,
|
||||
) => {
|
||||
const nodeId = generateId();
|
||||
const node = new Node({
|
||||
id: nodeId,
|
||||
type: "image",
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
fill: "transparent",
|
||||
stroke: "#333",
|
||||
strokeWidth: 2,
|
||||
draggable: true,
|
||||
imageUrl: renderedDataUrl,
|
||||
modelUrl,
|
||||
model3dParams: params,
|
||||
});
|
||||
node
|
||||
.setImageFromUrl(renderedDataUrl)
|
||||
.then(() => {
|
||||
get().addNode(node);
|
||||
})
|
||||
.catch((error) => console.error("Error loading 3D model render:", error));
|
||||
},
|
||||
|
||||
// Update the createImage method to handle both URLs and Files
|
||||
createImage: (
|
||||
x: number,
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
import * as THREE from "three";
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
||||
|
||||
export interface Model3DParams {
|
||||
cameraPosition: { x: number; y: number; z: number };
|
||||
cameraTarget: { x: number; y: number; z: number };
|
||||
fov: number;
|
||||
modelScale: number;
|
||||
/** Native render dimensions — the undistorted pixel size for this 3D model.
|
||||
* Stored once on first drop and preserved across re-edits. When the Konva
|
||||
* node is stretched, the overlay renders at these native dims and Konva
|
||||
* CSS-stretches the result to match the node's frame, preserving the stretch. */
|
||||
nativeWidth: number;
|
||||
nativeHeight: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_MODEL3D_PARAMS: Model3DParams = {
|
||||
cameraPosition: { x: 1.5, y: 1.5, z: 1.5 },
|
||||
cameraTarget: { x: 0, y: 0.4, z: 0 },
|
||||
fov: 50,
|
||||
modelScale: 1,
|
||||
nativeWidth: 512,
|
||||
nativeHeight: 512,
|
||||
};
|
||||
|
||||
function buildScene(): {
|
||||
scene: THREE.Scene;
|
||||
camera: THREE.PerspectiveCamera;
|
||||
renderer: THREE.WebGLRenderer;
|
||||
} {
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = null; // transparent
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(50, 1, 0.1, 1000);
|
||||
camera.position.set(1.5, 1.5, 1.5);
|
||||
camera.lookAt(0, 0.4, 0);
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
|
||||
renderer.setClearColor(0x000000, 0);
|
||||
|
||||
const ambientLight = new THREE.AmbientLight(0xffffff, 2);
|
||||
scene.add(ambientLight);
|
||||
|
||||
const hemisphereLight = new THREE.HemisphereLight(0xffffff, 0x888888, 1.2);
|
||||
scene.add(hemisphereLight);
|
||||
|
||||
const keyLight = new THREE.DirectionalLight(0xffffff, 2);
|
||||
keyLight.position.set(2, 10, 8);
|
||||
scene.add(keyLight);
|
||||
|
||||
const fillLight = new THREE.DirectionalLight(0xffffff, 1.2);
|
||||
fillLight.position.set(-6, 6, -4);
|
||||
scene.add(fillLight);
|
||||
|
||||
const frontLight = new THREE.DirectionalLight(0xffffff, 1);
|
||||
frontLight.position.set(0, 4, 10);
|
||||
scene.add(frontLight);
|
||||
|
||||
return { scene, camera, renderer };
|
||||
}
|
||||
|
||||
function normalizeAndPlaceModel(
|
||||
model: THREE.Object3D,
|
||||
modelScale: number,
|
||||
): void {
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
const maxDim = Math.max(size.x, size.y, size.z);
|
||||
const fitScale = (2 / maxDim) * modelScale;
|
||||
model.scale.multiplyScalar(fitScale);
|
||||
|
||||
const scaledBox = new THREE.Box3().setFromObject(model);
|
||||
const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
|
||||
model.position.x = -scaledCenter.x;
|
||||
model.position.z = -scaledCenter.z;
|
||||
model.position.y = -scaledBox.min.y;
|
||||
}
|
||||
|
||||
function disposeScene(
|
||||
scene: THREE.Scene,
|
||||
renderer: THREE.WebGLRenderer,
|
||||
): void {
|
||||
scene.traverse((obj) => {
|
||||
if (obj instanceof THREE.Mesh) {
|
||||
obj.geometry?.dispose();
|
||||
const mats = Array.isArray(obj.material) ? obj.material : [obj.material];
|
||||
mats.forEach((m) => m?.dispose());
|
||||
}
|
||||
});
|
||||
renderer.dispose();
|
||||
}
|
||||
|
||||
export async function render3DModelToDataUrl(
|
||||
modelUrl: string,
|
||||
params: Model3DParams,
|
||||
width = 512,
|
||||
height = 512,
|
||||
): Promise<string> {
|
||||
const { scene, camera, renderer } = buildScene();
|
||||
renderer.setSize(width, height);
|
||||
|
||||
camera.aspect = width / height;
|
||||
camera.fov = params.fov;
|
||||
camera.position.set(
|
||||
params.cameraPosition.x,
|
||||
params.cameraPosition.y,
|
||||
params.cameraPosition.z,
|
||||
);
|
||||
camera.lookAt(
|
||||
params.cameraTarget.x,
|
||||
params.cameraTarget.y,
|
||||
params.cameraTarget.z,
|
||||
);
|
||||
camera.updateProjectionMatrix();
|
||||
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const loader = new GLTFLoader();
|
||||
loader.load(
|
||||
modelUrl,
|
||||
(gltf) => {
|
||||
const model = gltf.scene;
|
||||
normalizeAndPlaceModel(model, params.modelScale);
|
||||
scene.add(model);
|
||||
|
||||
renderer.render(scene, camera);
|
||||
const dataUrl = renderer.domElement.toDataURL("image/png");
|
||||
|
||||
disposeScene(scene, renderer);
|
||||
resolve(dataUrl);
|
||||
},
|
||||
undefined,
|
||||
(error) => {
|
||||
disposeScene(scene, renderer);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user