Compare commits

...
9 changed files with 286 additions and 162 deletions
@@ -1,4 +1,4 @@
import { useState } from "react";
import { useContext, useEffect, useState } from "react";
import {
faChevronLeft,
faGear,
@@ -18,6 +18,7 @@ import { TabSelector, TabItem } from "@storyteller/ui-tab-selector";
import { Signal } from "@preact/signals-react";
import { useSignals } from "@preact/signals-react/runtime";
import { setLogoutStates } from "~/signals/authentication/utilities";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
function isEditorPath(path: string) {
if (path === "/") return true;
@@ -29,6 +30,7 @@ interface Props {
pageName: string;
appTabIdSignal: Signal<string>;
setAppTabId: (id: string) => void;
is3DInitSignal: Signal<boolean>;
}
const appTabs: TabItem[] = [
@@ -36,7 +38,7 @@ const appTabs: TabItem[] = [
{ id: "2D", label: "2D" },
];
export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
export const TopBar = ({ pageName, appTabIdSignal, setAppTabId, is3DInitSignal }: Props) => {
useSignals();
const currentLocation = getCurrentLocationWithoutParams(
@@ -47,6 +49,22 @@ export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
const [activeLibraryTab, setActiveLibraryTab] = useState("my-media");
const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
const engine3D = useContext(EngineContext);
const handleTabChange = (tabId: string) => {
if (appTabIdSignal.peek() == "3D") {
engine3D?.unmountEngine();
}
setAppTabId(tabId);
}
const currentAppTabId = appTabIdSignal.value;
const is3DInit = is3DInitSignal.value;
useEffect(() => {
if (currentAppTabId == "3D" && is3DInit) {
engine3D?.remountEngine();
}
}, [currentAppTabId, engine3D, is3DInit]);
return (
<>
<header className="fixed left-0 top-0 z-[60] w-full border-b border-white/5 bg-ui-background">
@@ -68,13 +86,13 @@ export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
Back to Editor
</ButtonLink>
)}
{/* <TabSelector
<TabSelector
tabs={appTabs}
activeTab={appTabIdSignal.value}
disabled={false}
onTabChange={(tabId: string) => setAppTabId(tabId)}
onTabChange={handleTabChange}
className="w-fit"
/> */}
/>
</div>
<div className="flex items-center justify-center gap-2 font-medium">
@@ -57,6 +57,7 @@ import { BufferType, EngineFrameBuffers } from "./VideoProcessor/engine_buffer";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
import { is3DEditorInitialized, setIs3DEditorInitialized } from "~/signals/appTab";
export type EditorInitializeConfig = {
sceneToken: string;
@@ -197,6 +198,11 @@ class Editor {
processingHasFailed: boolean;
stats: Stats;
// Allows us to cancel the queued render
private renderEventToken: number;
private shouldRender: boolean;
private isMounted: boolean;
constructor() {
this.processingHasFailed = false;
console.log(
@@ -256,6 +262,8 @@ class Editor {
this.lastFrameTime = 0;
this.last_selected_sum = 0;
this.selectedCanvas = false;
this.renderEventToken = -1;
this.shouldRender = false;
// Audio Engine Test.
this.render_camera_aspect_ratio = CameraAspectRatio.HORIZONTAL_3_2;
@@ -595,13 +603,6 @@ class Editor {
this.cam_obj = this.activeScene.get_object_by_name(this.camera_name);
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
this.timeline.characters,
);
if (result) outlinerState.items.value = result.items;
}
this.mouse_controls = new MouseControls(
this.camera,
this.get_camera_person_mode.bind(this),
@@ -642,10 +643,19 @@ class Editor {
this.mouse_controls.sceneManager = this.sceneManager;
}
// Creates the main update loop.
//this.renderer.setAnimationLoop(this.updateLoop.bind(this));
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
this.timeline.characters,
);
if (result) outlinerState.items.value = result.items;
}
this.updateLoop();
// Attach freecam state controller
this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
// This will enable all event and render loops
// We'll disable it here so the UI events can control is manually
// this.remountEngine();
if (!this.utils.isEmpty(sceneToken)) {
this.loadScene(sceneToken);
@@ -658,24 +668,6 @@ class Editor {
});
}
document.addEventListener("mouseover", (event) => {
if (this.cameraViewControls) {
if (
event.target instanceof HTMLCanvasElement ||
(event.target as HTMLElement).id == "letterbox"
) {
this.cameraViewControls.enabled = true;
this.selectedCanvas = true;
this.focused = true;
} else {
this.cameraViewControls.reset();
this.focused = false;
this.cameraViewControls.enabled = false;
this.selectedCanvas = false;
}
}
});
this._configurePostProcessingRaw();
loadingBarData.value = {
@@ -683,6 +675,8 @@ class Editor {
progress: 100,
};
loadingBarIsShowing.value = false;
setIs3DEditorInitialized(true);
}
public isMovable(): boolean {
@@ -693,6 +687,32 @@ class Editor {
document.body.appendChild(this.stats.dom);
}
private mouseOverEventHandler(event: MouseEvent) {
if (this.cameraViewControls) {
if (
event.target instanceof HTMLCanvasElement ||
(event.target as HTMLElement).id == "letterbox"
) {
this.cameraViewControls.enabled = true;
this.selectedCanvas = true;
this.focused = true;
} else {
this.cameraViewControls.reset();
this.focused = false;
this.cameraViewControls.enabled = false;
this.selectedCanvas = false;
}
}
}
public enableFreeCamControls() {
document.addEventListener("mouseover", this.mouseOverEventHandler);
}
public disableFreeCamControls() {
document.removeEventListener("mouseover", this.mouseOverEventHandler);
}
// Captures the scene without the grid
public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
if (!this.renderer?.domElement || !this.camera) {
@@ -1154,10 +1174,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 576
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
@@ -1166,10 +1186,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 1024
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 900
: 1000;
@@ -1376,10 +1396,15 @@ class Editor {
}
// Basicly Unity 3D's update loop.
async updateLoop() {
updateLoop() {
if (!this.shouldRender) {
console.debug("Stopping 3D render loop");
return;
}
// Performance improvement: Handle frame cap
// Request the next render already - this is necessary so the loop doesn't stop if the fps cap is hit
requestAnimationFrame(this.updateLoop.bind(this));
this.renderEventToken = requestAnimationFrame(this.updateLoop.bind(this));
const frameTime = performance.now();
if (frameTime - this.lastFrameTime < 1000 / this.cap_fps) {
return;
@@ -1389,6 +1414,49 @@ class Editor {
this.renderSingleFrame();
}
startRenderLoop() {
if (this.shouldRender) {
console.warn("Render flag is already true");
return;
}
this.shouldRender = true;
this.updateLoop();
}
stopRenderLoop() {
this.shouldRender = false;
if (this.renderEventToken) {
cancelAnimationFrame(this.renderEventToken);
this.renderEventToken = -1;
}
}
remountEngine() {
if (!is3DEditorInitialized.peek()) {
console.log("3D mount: Wait for initialization");
return;
}
if (this.isMounted) {
console.log("3D already mounted, skipping");
return;
}
this.startRenderLoop();
this.sceneManager?.attachEventListeners();
this.enableFreeCamControls();
this.cameraViewControls?.attachEventListeners();
}
unmountEngine() {
this.stopRenderLoop();
this.sceneManager?.detachEventListeners();
this.disableFreeCamControls();
this.cameraViewControls?.detachEventListeners();
}
change_mode(type: "translate" | "rotate" | "scale") {
if (this.control == undefined) {
return;
@@ -83,6 +83,10 @@ class FreeCam extends EventDispatcher {
this.keydown = this.keydown.bind(this);
this.keyup = this.keyup.bind(this);
this.mousedown = this.mousedown.bind(this);
this.mouseup = this.mouseup.bind(this);
this.mousemove = this.mousemove.bind(this);
this.mousewheel = this.mousewheel.bind(this);
this.reset = this.reset.bind(this);
this.update = this.update.bind(this);
this.updateMovementVector = this.updateMovementVector.bind(this);
@@ -94,18 +98,28 @@ class FreeCam extends EventDispatcher {
event.preventDefault();
});
window.addEventListener("keydown", this.keydown.bind(this));
window.addEventListener("keyup", this.keyup.bind(this));
window.addEventListener("mousedown", this.mousedown.bind(this));
window.addEventListener("mouseup", this.mouseup.bind(this));
window.addEventListener("mousemove", this.mousemove.bind(this));
window.addEventListener("wheel", this.mousewheel.bind(this));
this.updateMovementVector();
this.updateRotationVector();
}
attachEventListeners() {
window.addEventListener("keydown", this.keydown);
window.addEventListener("keyup", this.keyup);
window.addEventListener("mousedown", this.mousedown);
window.addEventListener("mouseup", this.mouseup);
window.addEventListener("mousemove", this.mousemove);
window.addEventListener("wheel", this.mousewheel);
}
detachEventListeners() {
window.removeEventListener("keydown", this.keydown);
window.removeEventListener("keyup", this.keyup);
window.removeEventListener("mousedown", this.mousedown);
window.removeEventListener("mouseup", this.mouseup);
window.removeEventListener("mousemove", this.mousemove);
window.removeEventListener("wheel", this.mousewheel);
}
keydown(event: KeyboardEvent) {
if (!this.enabled || isPromptBoxFocused.value) {
return;
@@ -65,6 +65,7 @@ export class SceneManager implements SceneManagerAPI {
private lastSceneState: SceneState;
private copiedObject: THREE.Object3D | undefined;
private is_character: Function;
private devMode: boolean;
constructor(
version: number,
@@ -80,14 +81,35 @@ export class SceneManager implements SceneManagerAPI {
this.lastSceneState = this.getSceneState();
this.updateOutliner = updateOutliner;
this.is_character = is_character;
this.devMode = devMode;
if (devMode) {
window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
window.addEventListener("click", this.onMouseClick.bind(this), false);
window.addEventListener("keydown", this.onKeyDown.bind(this), false);
window.addEventListener("mousedown", this.onMouseDown.bind(this), false);
window.addEventListener("mouseup", this.onMouseUp.bind(this), false);
this.onMouseMove = this.onMouseMove.bind(this);
this.onMouseClick = this.onMouseClick.bind(this);
this.onKeyDown = this.onKeyDown.bind(this);
this.onMouseDown = this.onMouseDown.bind(this);
this.onMouseUp = this.onMouseUp.bind(this);
}
public attachEventListeners() {
if (!this.devMode) {
return;
}
window.addEventListener("mousemove", this.onMouseMove, false);
window.addEventListener("click", this.onMouseClick, false);
window.addEventListener("keydown", this.onKeyDown, false);
window.addEventListener("mousedown", this.onMouseDown, false);
window.addEventListener("mouseup", this.onMouseUp, false);
}
public detachEventListeners() {
if (!this.devMode) {
return;
}
window.removeEventListener("mousemove", this.onMouseMove, false);
window.removeEventListener("click", this.onMouseClick, false);
window.removeEventListener("keydown", this.onKeyDown, false);
window.removeEventListener("mousedown", this.onMouseDown, false);
window.removeEventListener("mouseup", this.onMouseUp, false);
}
public async undo() {
@@ -121,7 +143,7 @@ export class SceneManager implements SceneManagerAPI {
name: string,
position: THREE.Vector3,
): Promise<THREE.Object3D<THREE.Object3DEventMap>> {
if (media_token.includes("SKY::")){
if (media_token.includes("SKY::")) {
const token = media_token.replace("SKY::", "");
this.scene.updateSkybox(token);
}
@@ -243,16 +265,16 @@ export class SceneManager implements SceneManagerAPI {
}
public async copy() {
// TODO MAKE BETTER FIX: Temp disables copy and paste of characters.
// TODO MAKE BETTER FIX: Temp disables copy and paste of characters.
const object = this.mouse_controls.selected?.at(0)
if(object !== undefined ){
if(this.is_character(object.uuid) === false)
this.copiedObject = object;
if (object !== undefined) {
if (this.is_character(object.uuid) === false)
this.copiedObject = object;
}
}
public async paste() {
if(this.copiedObject && this.copiedObject.name != "::CAM::") {
if (this.copiedObject && this.copiedObject.name != "::CAM::") {
const userdata = this.copiedObject.userData;
const position = this.copiedObject.position.clone();
const rotation = this.copiedObject.rotation.clone();
@@ -264,7 +286,7 @@ export class SceneManager implements SceneManagerAPI {
const obj = await this.create(media_id, name, position);
this.scene.setColor(obj.uuid, color);
obj.position.copy(position.add(new THREE.Vector3(0.5,0.0,0.5)));
obj.position.copy(position.add(new THREE.Vector3(0.5, 0.0, 0.5)));
obj.rotation.copy(rotation);
obj.scale.copy(scale);
@@ -28,7 +28,7 @@ import { PromptBox3D } from "@storyteller/ui-promptbox";
import { PopoverItem } from "@storyteller/ui-popover";
import { OnboardingHelper } from "./comps/OnboardingHelper";
import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDisplay";
import { appTabId, setAppTabId } from "~/signals/appTab";
import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
import { KonvaCanvasContainer } from "../Page2d/KonvaCanvasContainer";
import { KonvaRootComponent } from "../Page2d/KonvaRootComponent";
import {
@@ -251,13 +251,8 @@ export const PageEditor = () => {
});
};
const ContentFor2D = () => {
return (
<KonvaCanvasContainer>
<KonvaRootComponent className="h-full w-full" />
</KonvaCanvasContainer>
);
};
const display3d = appTabId.value === "3D" ? "block" : "none";
const display2d = appTabId.value === "2D" ? "block" : "none";
return (
<div className="w-screen">
@@ -265,109 +260,109 @@ export const PageEditor = () => {
pageName="Edit Scene"
appTabIdSignal={appTabId}
setAppTabId={setAppTabId}
is3DInitSignal={is3DEditorInitialized}
/>
{appTabId.value === "3D" ? (
<>
<OnboardingHelper />
<>
<OnboardingHelper />
<div
className="relative flex w-screen"
style={{ height: "calc(100vh - 68px)", display: display3d }}
>
{/* Engine section/side panel */}
<div
className="relative flex w-screen"
style={{ height: "calc(100vh - 68px)" }}
id="engine-n-panels-wrapper"
className="flex"
style={{
height,
}}
>
{/* Engine section/side panel */}
<div
id="engine-n-panels-wrapper"
className="flex"
style={{
height,
}}
>
<div className="relative w-full overflow-hidden bg-transparent">
<SceneContainer>
<EditorCanvas />
<PreviewFrameImage />
</SceneContainer>
<div className="relative w-full overflow-hidden bg-transparent">
<SceneContainer>
<EditorCanvas />
<PreviewFrameImage />
</SceneContainer>
{/* Focal Length Display */}
<FocalLengthDisplay />
{/* Focal Length Display */}
<FocalLengthDisplay />
{/* Pose Mode Selector */}
<PoseModeSelector />
{/* Pose Mode Selector */}
<PoseModeSelector />
{/* Top controls */}
<div
className="absolute left-0 top-0 w-full"
onClick={handleOverlayClick}
>
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
</div>
{/* Top controls */}
<div
className="absolute left-0 top-0 w-full"
onClick={handleOverlayClick}
>
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
</div>
{/* Bottom controls */}
<div
className="absolute bottom-0 left-0"
style={{
width: pageWidth.value,
}}
onClick={handleOverlayClick}
>
<div
className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
style={{ transform: `scale(${getScale()})` }}
>
<Outliner />
<PreviewEngineCamera />
</div>
<ControlPanelSceneObject />
</div>
<PromptBox3D
cameras={cameras}
cameraAspectRatio={cameraAspectRatio}
disableHotkeyInput={disableHotkeyInput}
enableHotkeyInput={enableHotkeyInput}
gridVisibility={gridVisibility}
setGridVisibility={setGridVisibility}
selectedCameraId={selectedCameraId}
deleteCamera={deleteCamera}
focalLengthDragging={focalLengthDragging}
isPromptBoxFocused={isPromptBoxFocused}
uploadImage={uploadImage}
handleCameraSelect={handleCameraSelect}
handleAddCamera={handleAddCamera}
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
setEnginePrompt={(prompt) => {
console.log("setEnginePrompt", prompt);
if (!editorEngine) {
console.log("editorEngine is not available");
return;
}
editorEngine!.positive_prompt = prompt;
}}
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
/>
<LoadingDots
className="absolute left-0 top-0 z-50"
isShowing={editorLoader.value.isShowing}
type="bricks"
message={editorLoader.value.message}
/>
</div>
{/* Bottom controls */}
<div
className="absolute bottom-0 left-0"
style={{
width: pageWidth.value,
}}
onClick={handleOverlayClick}
>
<div
className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
style={{ transform: `scale(${getScale()})` }}
>
<Outliner />
<PreviewEngineCamera />
</div>
<ControlPanelSceneObject />
</div>
<PromptBox3D
cameras={cameras}
cameraAspectRatio={cameraAspectRatio}
disableHotkeyInput={disableHotkeyInput}
enableHotkeyInput={enableHotkeyInput}
gridVisibility={gridVisibility}
setGridVisibility={setGridVisibility}
selectedCameraId={selectedCameraId}
deleteCamera={deleteCamera}
focalLengthDragging={focalLengthDragging}
isPromptBoxFocused={isPromptBoxFocused}
uploadImage={uploadImage}
handleCameraSelect={handleCameraSelect}
handleAddCamera={handleAddCamera}
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
setEnginePrompt={(prompt) => {
console.log("setEnginePrompt", prompt);
if (!editorEngine) {
console.log("editorEngine is not available");
return;
}
editorEngine!.positive_prompt = prompt;
}}
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
/>
<LoadingDots
className="absolute left-0 top-0 z-50"
isShowing={editorLoader.value.isShowing}
type="bricks"
message={editorLoader.value.message}
/>
</div>
</div>
</>
) : (
<ContentFor2D />
)}
</div>
</>
<KonvaCanvasContainer style={{ display: display2d }}>
<KonvaRootComponent className="h-full w-full" />
</KonvaCanvasContainer>
<LoginModal
onClose={() => {}}
onClose={() => { }}
videoSrc2D="/resources/videos/artcraft-canvas-demo.mp4"
videoSrc3D="/resources/videos/artcraft-3d-demo.mp4"
openAiLogo="/resources/images/openai-logo.png"
@@ -18,7 +18,7 @@ import { StyleSelection } from "~/pages/PageEnigma/comps/StyleSelection";
import { StyleMoreOptions } from "~/pages/PageEnigma/comps/StyleMoreOptions/StyleMoreOptions";
import Queue, { QueueNames } from "~/pages/PageEnigma/Queue";
import { toTimelineActions } from "~/pages/PageEnigma/Queue/toTimelineActions";
import { appTabId, setAppTabId } from "~/signals/appTab";
import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
export const PageStyling = () => {
useSignals();
@@ -64,6 +64,7 @@ export const PageStyling = () => {
pageName="Stylization"
appTabIdSignal={appTabId}
setAppTabId={setAppTabId}
is3DInitSignal={is3DEditorInitialized}
/>
<div className="mt-4 flex flex-col items-center gap-6">
<PreviewImages />
@@ -10,3 +10,8 @@ export const setAppTabId = (newId: string) => {
appTabId.value = newId;
};
export const is3DEditorInitialized = signal(false);
export const setIs3DEditorInitialized = (isInitialized: boolean) => {
is3DEditorInitialized.value = isInitialized;
}
+2 -1
View File
@@ -24,9 +24,10 @@
"allowJs": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"baseUrl": ".",
"paths": {
"~/*": [
"apps/editor3d/app/src/*"
"app/src/*"
]
},
// Vite takes care of building everything, not tsc.
@@ -110,7 +110,7 @@ export const PromptBox2D = ({
textareaRef.current.style.height = "auto";
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
}
}, [prompt]);
});
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatioList((prev) =>