Compare commits

...
Author SHA1 Message Date
Brandon Thomas 14feaccb91 Merge branch 'main' into feature/generatemode 2025-03-25 01:38:48 -04:00
Brandon Thomas e1dbf4f2fe add inpaint demo/test endpoint 2025-03-22 03:16:48 -04:00
Tanish Manku d826db599d Remove the cancel generation button for now 2025-03-22 00:08:06 -07:00
Tanish Manku a642732c59 Clip overflow to have top rounded corners on image 2025-03-22 00:02:21 -07:00
Brandon Thomas 435c5aa0b4 fix strength bug 2025-03-22 02:57:06 -04:00
Tanish Manku 90b4aff846 Fix: Multiple invokes of a prompt after changing mode 2025-03-21 23:56:50 -07:00
Tanish Manku 4d3f097432 Hook up the TTI UI to invoke call 2025-03-21 23:38:25 -07:00
Tanish Manku e5de0e4731 Merge branch 'main' into feature/generatemode 2025-03-21 23:11:07 -07:00
Tanish Manku 04498b0457 Add initial Edit mode files 2025-03-21 23:02:17 -07:00
bflatastic fbfc1e15f2 gallery gap fix 2025-03-21 22:46:30 +07:00
bflatastic 4a65317729 rename image to realtime 2025-03-21 20:13:13 +07:00
Tanish Manku e7ed0693a3 Add generation engine and signals 2025-03-21 00:44:44 -07:00
Tanish Manku 99ac8faf33 Merge branch 'main' into feature/generatemode 2025-03-20 23:41:19 -07:00
bflatastic f0213d4a63 cancel button 2025-03-21 00:03:15 +07:00
bflatastic 58f684b1e0 tab disabled prop and disabled message to block tab change when generating 2025-03-20 22:42:36 +07:00
bflatastic e83389a874 background gallery + many other styling and animation transition updates 2025-03-20 22:11:12 +07:00
Tanish Manku 3602f8fc2b Add generation signals 2025-03-20 02:10:27 -07:00
Tanish Manku 115c4fc60c Add a generation mode UI component 2025-03-19 23:58:53 -07:00
45 changed files with 1000 additions and 27 deletions
@@ -0,0 +1,24 @@
use crate::ml::model_cache::ModelCache;
use crate::ml::prompt_cache::PromptCache;
use crate::state::app_config::AppConfig;
use crate::state::app_dir::AppDataRoot;
use image::{ImageReader, RgbImage};
use log::info;
use tauri::{AppHandle, State};
#[tauri::command]
pub async fn inpaint_image(
image: &str,
mask: &str,
prompt: String,
model_config: State<'_, AppConfig>,
model_cache: State<'_, ModelCache>,
prompt_cache: State<'_, PromptCache>,
app_data_root: State<'_, AppDataRoot>,
app: AppHandle,
) -> Result<String, String> {
info!("inpaint_endpoint endpoint called.");
Ok(mask.to_string())
}
@@ -1,5 +1,6 @@
pub mod download_models;
pub mod flip_image;
pub mod inpaint_image_endpoint;
pub mod realtime_image_endpoint;
pub mod remove_background_endpoint;
pub mod text_to_image_endpoint;
@@ -17,7 +17,7 @@ use tauri::{AppHandle, State};
const RANDOM_SEED: u32 = 42;
const STRENGTH: f64 = 1.0;
const STRENGTH: f64 = 100.0;
const PNG_BYTES : &[u8] = include_bytes!("../../binary_includes/1024.png");
// NB: We're just hacking a temporary "text to image" test endpoint leveraging the existing image to image modality.
+2
View File
@@ -7,6 +7,7 @@ pub mod transfer;
pub mod utils;
use crate::endpoints::download_models::download_models;
use crate::endpoints::inpaint_image_endpoint::inpaint_image;
use crate::endpoints::flip_image::flip_image;
use crate::endpoints::realtime_image_endpoint::infer_image;
use crate::endpoints::remove_background_endpoint::remove_background;
@@ -71,6 +72,7 @@ pub fn run() {
download_models,
flip_image,
infer_image,
inpaint_image,
remove_background,
text_to_image,
])
+18 -6
View File
@@ -8,6 +8,7 @@
"name": "storyteller-board",
"version": "0.0.0",
"dependencies": {
"@devnomic/marquee": "^1.0.3",
"@fortawesome/free-brands-svg-icons": "^6.6.0",
"@headlessui/react": "^2.1.3",
"@preact/signals-react": "^2.2.0",
@@ -381,6 +382,21 @@
"node": ">=6.9.0"
}
},
"node_modules/@devnomic/marquee": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@devnomic/marquee/-/marquee-1.0.3.tgz",
"integrity": "sha512-q8QUtNO8ff+UvfrKJsHBtxh4fbOfhGk6SSdK0Uu/XZFdEzrmiGJ/l0FuVHRYsPAmzgnhVVnz1dGNB5CWGLMVvw==",
"dependencies": {
"clsx": "^2.1.0",
"tailwind-merge": "^2.2.0"
},
"peerDependencies": {
"@types/react": "^18.2.47",
"@types/react-dom": "^18.2.18",
"react": "^18.2.0",
"react-dom": "^18.2.0"
}
},
"node_modules/@emotion/is-prop-valid": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.3.0.tgz",
@@ -1510,14 +1526,12 @@
"node_modules/@types/prop-types": {
"version": "15.7.12",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz",
"integrity": "sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==",
"dev": true
"integrity": "sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q=="
},
"node_modules/@types/react": {
"version": "18.3.4",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.4.tgz",
"integrity": "sha512-J7W30FTdfCxDDjmfRM+/JqLHBIyl7xUIp9kwK637FGmY7+mkSFSe6L4jpZzhj5QMfLssSDP4/i75AKkrdC7/Jw==",
"dev": true,
"dependencies": {
"@types/prop-types": "*",
"csstype": "^3.0.2"
@@ -1527,7 +1541,6 @@
"version": "18.3.0",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.0.tgz",
"integrity": "sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg==",
"dev": true,
"dependencies": {
"@types/react": "*"
}
@@ -2364,8 +2377,7 @@
"node_modules/csstype": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"dev": true
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="
},
"node_modules/data-view-buffer": {
"version": "1.0.1",
+1
View File
@@ -11,6 +11,7 @@
"preview": "vite preview"
},
"dependencies": {
"@devnomic/marquee": "^1.0.3",
"@fortawesome/free-brands-svg-icons": "^6.6.0",
"@headlessui/react": "^2.1.3",
"@preact/signals-react": "^2.2.0",
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 276 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 320 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 564 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 366 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 452 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 337 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 378 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 302 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 446 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 238 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 344 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 299 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 396 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 318 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 521 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 318 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 396 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 316 KiB

@@ -0,0 +1,25 @@
import { useCallback, useRef } from "react";
import { useRenderCounter } from "~/hooks/useRenderCounter";
import { EditEngine } from "~/KonvaApp/EditEngine";
import { KonvaCanvasContainer } from "~/KonvaRootComponent/KonvaCanvasContainer";
export const EditModeRootComponent = ({ className }: { className: string }) => {
useRenderCounter("EditModeRootComponent");
const editEngineRef = useRef<EditEngine | null>(null);
const konvaContainerCallbackRef = useCallback((node: HTMLDivElement) => {
if (node !== null && editEngineRef.current === null) {
editEngineRef.current = new EditEngine(node);
}
}, []);
return (
<>
<KonvaCanvasContainer
className={className}
ref={konvaContainerCallbackRef}
/>
</>
);
};
@@ -0,0 +1,145 @@
import { useEffect, useState } from "react";
import { Marquee } from "@devnomic/marquee";
import "@devnomic/marquee/dist/index.css";
function getRandomImages(count: number) {
const images = Array.from({ length: count }, (_, i) => ({
id: i + 1,
src: `/images/generate/image${i + 1}.png`,
alt: `Generated image ${i + 1}`,
}));
// Shuffle the array
return [...images].sort(() => Math.random() - 0.5);
}
interface ImageColumnProps {
images: Array<{ id: number; src: string; alt: string }>;
delay: number;
reverse?: boolean;
isLoaded: boolean;
duration?: number;
}
function ImageColumn({
images,
delay,
reverse = false,
duration = 100,
isLoaded,
}: ImageColumnProps) {
return (
<div className="flex h-screen flex-col gap-6">
<Marquee
fade={false}
direction="up"
reverse={reverse}
pauseOnHover={false}
className={`h-full gap-[24px]`}
style={{ "--duration": `${duration}s` } as React.CSSProperties}
innerClassName="gap-[24px] [--gap:24px]"
numberOfCopies={2}
>
{images.map((image, i) => (
<div
key={`${image.id}-${i}`}
className={`relative aspect-[4/5] w-full overflow-hidden rounded-lg shadow-lg transition-all duration-700 ease-out ${
isLoaded
? "translate-y-0 opacity-100"
: "translate-y-12 opacity-0"
}`}
style={{
transitionDelay: `${delay + i * 50}ms`,
}}
>
<img
src={image.src}
alt={image.alt}
className="w-full object-cover"
/>
</div>
))}
</Marquee>
</div>
);
}
export default function GalleryBackground() {
const [images, _setImages] = useState(() => getRandomImages(20));
const [isLoaded, setIsLoaded] = useState(false);
useEffect(() => {
// delay for smooth animation
const timer = setTimeout(() => {
setIsLoaded(true);
}, 100);
return () => clearTimeout(timer);
}, []);
// Columns for masonry layout
const columns = {
col1: images.filter((_, i) => i % 5 === 0),
col2: images.filter((_, i) => i % 5 === 1),
col3: images.filter((_, i) => i % 5 === 2),
col4: images.filter((_, i) => i % 5 === 3),
col5: images.filter((_, i) => i % 5 === 4),
};
return (
<>
<div className="fixed inset-0 z-[1] overflow-hidden bg-[radial-gradient(50%_50%_at_50%_50%,rgba(0,0,0,0.00)_49%,rgba(0,0,0,0.60)_100%)]" />
<div className="fixed inset-0 z-0 overflow-hidden">
<div
className={`h-full w-full transition-opacity duration-1000 ${isLoaded ? "opacity-50" : "opacity-0"}`}
>
<div className="absolute inset-0 grid grid-cols-5 gap-6 px-6 opacity-35">
<ImageColumn
images={columns.col1}
delay={0}
isLoaded={isLoaded}
duration={100}
/>
<ImageColumn
images={columns.col2}
delay={columns.col1.length * 50}
isLoaded={isLoaded}
reverse={true}
duration={100}
/>
<ImageColumn
images={columns.col3}
delay={(columns.col1.length + columns.col2.length) * 50}
isLoaded={isLoaded}
duration={125}
/>
<ImageColumn
images={columns.col4}
delay={
(columns.col1.length +
columns.col2.length +
columns.col3.length) *
50
}
isLoaded={isLoaded}
reverse={true}
duration={160}
/>
<ImageColumn
images={columns.col5}
delay={
(columns.col1.length +
columns.col2.length +
columns.col3.length +
columns.col4.length) *
50
}
isLoaded={isLoaded}
duration={110}
/>
</div>
</div>
</div>
</>
);
}
@@ -0,0 +1,196 @@
import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
import { useSignals } from "@preact/signals-react/runtime";
import { Button, LoadingSpinner } from "~/components/ui";
import { useRenderCounter } from "~/hooks/useRenderCounter";
import { GenerationLoadingState, generationSignal } from "~/signals";
import { useRef, useState } from "react";
import BackgroundGallery from "./BackgroundGallery";
import { twMerge } from "tailwind-merge";
import { Transition } from "@headlessui/react";
import { GenerationEngine } from "~/KonvaApp/GenerationEngine";
import { ensureBase64Prefix } from "~/KonvaApp/EngineUtitlities/Base64Helpers";
export const GenerationRootComponent = ({
generationEngineRef,
}: {
generationEngineRef: React.MutableRefObject<GenerationEngine | null>;
}) => {
// This is a hook that will log the number of times the component has rerendered
// Let's make sure we only log once
useRenderCounter("GenerationRootComponent");
useSignals();
const generationState = generationSignal.value;
const inputRef = useRef<HTMLTextAreaElement>(null);
const [isInputFocused, setIsInputFocused] = useState(false);
if (generationEngineRef.current === null) {
generationEngineRef.current = new GenerationEngine();
}
const handleGenerate = () => {
if (!inputRef.current) {
return;
}
const prompt = inputRef.current.value;
generationSignal.value = {
loadingState: GenerationLoadingState.GENERATING,
prompt,
};
};
let contentElement;
switch (generationState.loadingState) {
case GenerationLoadingState.INIT:
contentElement = null;
break;
case GenerationLoadingState.GENERATING:
case GenerationLoadingState.GENERATED:
contentElement = (
<GenerationContent
loadingState={generationState.loadingState}
imageB64={generationState.imageB64}
prompt={generationState.prompt}
/>
);
break;
}
// const isButtonDisabled = generationState.loadingState === GenerationLoadingState.GENERATING;
const isButtonDisabled = false;
return (
<>
<div className="fixed z-10 flex h-full w-full flex-col items-center justify-center transition-all duration-500">
<Transition
show={generationState.loadingState !== GenerationLoadingState.INIT}
enter="transition-opacity duration-500"
enterFrom="opacity-0"
enterTo="opacity-100"
leave="transition-opacity duration-500"
leaveFrom="opacity-100"
leaveTo="opacity-0"
as="div"
>
{contentElement}
</Transition>
<div
className={twMerge(
"absolute flex w-full flex-col items-center justify-center gap-16 transition-all duration-500",
generationState.loadingState === GenerationLoadingState.INIT
? "bottom-1/2 translate-y-1/2 transform"
: "bottom-8",
)}
>
<Transition
show={generationState.loadingState === GenerationLoadingState.INIT}
enter="transition-opacity duration-200"
enterFrom="opacity-0"
enterTo="opacity-100"
leave="transition-opacity duration-200"
leaveFrom="opacity-100"
leaveTo="opacity-0"
as="div"
>
<div className="flex flex-col items-center justify-center text-center drop-shadow-xl">
<span className="text-8xl font-bold">Generate Image</span>
<span className="pt-2 text-2xl opacity-80">
Imagine, Describe, Generate.
</span>
</div>
</Transition>
<div
className={twMerge(
"glass flex min-h-[56px] w-full max-w-[900px] items-end rounded-xl border-2 p-3 shadow-xl transition-all duration-[400ms] ease-in-out",
isInputFocused ? "border-primary-400/60" : "border-ui-panel",
)}
>
<textarea
ref={inputRef}
rows={1}
placeholder="Describe what you want to see..."
className="max-h-[120px] min-h-[40px] flex-1 resize-none overflow-y-auto rounded-md bg-transparent px-3 py-2 text-lg leading-normal focus:outline-none"
style={{ lineHeight: "24px" }}
onInput={(e) => {
const target = e.target as HTMLTextAreaElement;
target.style.height = "auto";
target.style.height = `${target.scrollHeight}px`;
}}
autoFocus={true}
onFocus={() => setIsInputFocused(true)}
onBlur={() => setIsInputFocused(false)}
/>
<Button
icon={faSparkles}
variant="primary"
className="text-md ml-2 self-end"
onClick={handleGenerate}
disabled={isButtonDisabled}
loading={
generationState.loadingState ===
GenerationLoadingState.GENERATING
}
>
Generate
</Button>
</div>
</div>
</div>
<Transition
show={generationState.loadingState === GenerationLoadingState.INIT}
enter="transition-opacity duration-700"
enterFrom="opacity-0"
enterTo="opacity-100"
leave="transition-opacity duration-700"
leaveFrom="opacity-100"
leaveTo="opacity-0"
as="div"
>
<BackgroundGallery />
</Transition>
</>
);
};
const GenerationContent = (generationState: {
loadingState: GenerationLoadingState;
imageB64?: string;
prompt: string;
}) => {
let imgBoxContent;
if (generationState.loadingState === GenerationLoadingState.GENERATING) {
imgBoxContent = (
<div className="flex h-full w-full flex-col items-center justify-center gap-7">
<LoadingSpinner isShowing={true} message="Generating image..." />
</div>
);
} else if (
generationState.loadingState === GenerationLoadingState.GENERATED &&
generationState.imageB64
) {
imgBoxContent = (
<img
src={ensureBase64Prefix(generationState.imageB64)}
alt="Generated Image"
className="h-full w-full object-contain"
/>
);
}
return (
<>
<div className="absolute inset-0 flex h-full w-full flex-col items-center justify-center">
<div className="aspect-[1/1] w-[min(80vw,68vh)] overflow-clip rounded-t-lg bg-[#29292D]/60">
{imgBoxContent}
</div>
<div className="mb-16 flex w-[min(80vw,68vh)] rounded-b-lg bg-[#29292D]">
<span className="p-3.5 text-sm opacity-50">
{generationState.prompt || "No prompt provided"}
</span>
</div>
</div>
</>
);
};
@@ -0,0 +1,365 @@
import Konva from "konva";
import { uiAccess } from "~/signals";
import { CommandManager, MatteBox } from "./EngineUtitlities";
import { ImageNode } from "./Nodes";
import { PaintNode } from "./Nodes/PaintNode";
import {
NodesManager,
NodesTranslationEventDetails,
NodeTransformationEventDetails,
NodeTransformer,
SelectionManager,
SelectionManagerEvents,
SelectorSquare,
} from "./NodesManagers";
import { UndoStackManager } from "./UndoRedo";
import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
import { EditModes, VideoResolutions } from "./constants";
import { ToolbarNodeButtonNames } from "~/components/features/ToolbarNode/enums";
import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine";
export interface RenderingOptions {
artstyle: string;
positivePrompt: string;
negativePrompt: string;
cinematic: boolean;
enginePreProcessing: boolean;
faceDetail: boolean;
lipSync: boolean;
upscale: boolean;
styleStrength: number;
}
export class EditEngine {
private editMode: EditModes = EditModes.SELECT;
private boardCanvasRef: HTMLDivElement;
private stage: Konva.Stage;
private bgLayer: Konva.Layer;
private mediaLayer: Konva.Layer;
private nodeIsolationLayer: Konva.Layer;
private uiLayer: Konva.Layer;
private offScreenCanvas: OffscreenCanvas;
private realTimeDrawEngine: RealTimeDrawEngine;
private nodesManager: NodesManager;
private nodeTransformer: NodeTransformer;
private selectionManager: SelectionManager;
private selectorSquare: SelectorSquare;
private matteBox: MatteBox;
private undoStackManager: UndoStackManager;
private commandManager: CommandManager;
public segmentationButtonCanBePressed: boolean = true;
// signal reference
constructor(boardCanvasRef: HTMLDivElement) {
if (import.meta.env.DEV) {
console.log("Engine Created");
}
this.editMode = EditModes.SELECT;
this.boardCanvasRef = boardCanvasRef;
this.stage = new Konva.Stage({
container: this.boardCanvasRef,
width: window.innerWidth,
height: window.innerHeight,
});
this.bgLayer = new Konva.Layer();
this.mediaLayer = new Konva.Layer();
this.nodeIsolationLayer = new Konva.Layer();
this.uiLayer = new Konva.Layer();
this.stage.add(this.bgLayer);
this.stage.add(this.mediaLayer);
this.stage.add(this.nodeIsolationLayer);
this.stage.add(this.uiLayer);
// Konva Transformer
this.nodeTransformer = new NodeTransformer();
this.uiLayer.add(this.nodeTransformer.getKonvaNode());
// core layer for all the work done.
this.offScreenCanvas = new OffscreenCanvas(0, 0);
// Collection of all Nodes
this.nodesManager = new NodesManager();
// Partial Collection of selected Nodes
this.selectionManager = new SelectionManager({
nodeTransformerRef: this.nodeTransformer,
mediaLayerRef: this.mediaLayer,
});
this.realTimeDrawEngine = new RealTimeDrawEngine({
width: VideoResolutions.SQUARE_1024.width,
height: VideoResolutions.SQUARE_1024.height,
mediaLayerRef: this.mediaLayer,
offScreenCanvas: this.offScreenCanvas,
onDraw: async (canvas, lineBounds) => {
await this.addPaintNode(canvas, lineBounds);
},
});
// Selector Square to select Nodes
this.selectorSquare = new SelectorSquare({
captureCanvasRef: this.realTimeDrawEngine.captureCanvas,
mediaLayerRef: this.mediaLayer,
nodesManagerRef: this.nodesManager,
selectionManagerRef: this.selectionManager,
stageRef: this.stage,
});
this.uiLayer.add(this.selectorSquare.getKonvaNode());
//Collection of commands for undo-redo
this.undoStackManager = new UndoStackManager(() => {
this.realTimeDrawEngine.render();
});
this.commandManager = new CommandManager({
mediaLayerRef: this.mediaLayer,
nodesManagerRef: this.nodesManager,
nodeTransformerRef: this.nodeTransformer,
selectionManagerRef: this.selectionManager,
renderEngineRef: this.realTimeDrawEngine,
undoStackManagerRef: this.undoStackManager,
});
this.matteBox = new MatteBox({
boardCanvasSize: {
width: this.boardCanvasRef.clientWidth,
height: this.boardCanvasRef.clientHeight,
},
captureCanvasSize: this.realTimeDrawEngine.captureCanvas.getSize(),
uiLayerRef: this.uiLayer,
});
// some of the managers has events
// hence, lastly, setup these events
this.setupEventSystem();
this.setEditMode(this.editMode);
}
private setEditMode(newEditMode: EditModes) {
this.editMode = newEditMode;
switch (this.editMode) {
case EditModes.SELECT:
default: {
console.log("EDITMODE: SELECT");
this.realTimeDrawEngine.disablePaintMode();
this.selectorSquare.enable();
this.selectionManager.enable();
uiAccess.toolbarMain.enable();
uiAccess.toolbarMain.changeButtonState(ToolbarMainButtonNames.SELECT, {
active: true,
});
uiAccess.toolbarMain.changeButtonState(ToolbarMainButtonNames.PAINT, {
active: false,
});
this.matteBox.disable();
document.body.style.cursor = "default";
return;
}
case EditModes.EDIT: {
console.log("EDITMODE: EDIT");
this.realTimeDrawEngine.enablePaintMode();
this.selectorSquare.disable();
this.selectionManager.disable();
}
}
}
private setupEventSystem() {
// Listen to changes in container size
const resizeObserver = new ResizeObserver(() => {
this.onBoardCanvasResize();
});
resizeObserver.observe(this.boardCanvasRef);
this.onBoardCanvasResize();
// Listen to Nodes
this.selectionManager.eventTarget.addEventListener(
SelectionManagerEvents.NODES_TRANSLATIONS,
((event: CustomEvent<NodesTranslationEventDetails>) => {
//console.log("Event: SelectionManager -> Engine", event);
this.commandManager.translateNodes(event.detail);
}) as EventListener,
);
this.selectionManager.eventTarget.addEventListener(
SelectionManagerEvents.NODES_TRANSFORMATION,
((event: CustomEvent<NodeTransformationEventDetails>) => {
//console.log("Event: SelectionManager -> Engine", event);
this.commandManager.transformNodes(event.detail);
}) as EventListener,
);
// Listen to Toolbar Main
// uiEvents.toolbarMain.UNDO.onClick(() => {
// this.undoStackManager.undo();
// });
// uiEvents.toolbarMain.REDO.onClick(() => {
// this.undoStackManager.redo();
// this.realTimeDrawEngine.render();
// });
// uiEvents.toolbarMain.SAVE.onClick(async (/*event*/) => {
// await this.realTimeDrawEngine.saveOutput();
// });
// uiEvents.toolbarMain.SELECT.onClick(() => {
// console.log("Toolbar Main >> Select");
// this.setAppMode(AppModes.SELECT);
// });
// uiEvents.toolbarMain.PAINT.onClick(() => {
// console.log("Toolbar Main >> Paint");
// this.setAppMode(AppModes.PAINT);
// });
}
disableAllButtons() {
const buttonNames = Object.values(ToolbarNodeButtonNames);
for (const name of buttonNames) {
uiAccess.toolbarNode.changeButtonState(name, { disabled: true });
}
}
async enableAllButtons() {
const buttonNames = Object.values(ToolbarNodeButtonNames);
for (const name of buttonNames) {
await uiAccess.toolbarNode.changeButtonState(name, { disabled: false });
}
}
sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
private onBoardCanvasResize() {
this.realTimeDrawEngine.updateCaptureCanvas(undefined, undefined);
this.matteBox.updateSize({
boardCanvasSize: {
width: this.boardCanvasRef.offsetWidth,
height: this.boardCanvasRef.offsetHeight,
},
});
this.uiLayer.draw();
this.stage.width(this.boardCanvasRef.offsetWidth);
this.stage.height(this.boardCanvasRef.offsetHeight);
this.stage.draw(); // Redraw the canvas
}
// Sandbox is quickly a way to test your idea.
public async sandbox() {}
public onMessage(event: MessageEvent) {
console.log("Message From Shared Worker");
console.log(event);
}
public initializeStage() {
// load canvas that was originaly saved TODO Save manager for resharing.
uiAccess.toolbarNode.hide();
uiAccess.loadingBar.hide();
this.setupStage();
}
public isInitialized() {
return this.stage !== null;
}
public async setupStage() {
// Frame rate inicator
const textNode = new Konva.Text({
x: 10,
y: 80,
text: "",
fontSize: 18,
fontFamily: "Source Sans 3",
fill: "white",
});
const anim = new Konva.Animation((frame) => {
if (frame && import.meta.env.DEV) {
const timeDiff = frame.timeDiff;
const frameRate = frame.frameRate;
textNode.setText(
`FrameTime: ${timeDiff.toFixed(0)} ms\nFrameRate: ${frameRate.toFixed(0)} fps`,
);
}
}, this.mediaLayer);
anim.start();
this.uiLayer.add(textNode);
this.addKeyboardShortcuts();
}
public addImage(imageFile: File) {
const imageNode = new ImageNode({
mediaLayerRef: this.mediaLayer,
canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
imageFile: imageFile,
selectionManagerRef: this.selectionManager,
loaded: async () => {
await this.realTimeDrawEngine.render();
this.setEditMode(EditModes.SELECT);
},
});
this.commandManager.createNode(imageNode);
}
public addImageFromImageBitmap(imageBitmap: ImageBitmap) {
const imageNode = new ImageNode({
mediaLayerRef: this.mediaLayer,
canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
imageBitmap: imageBitmap,
selectionManagerRef: this.selectionManager,
loaded: async () => {
await this.realTimeDrawEngine.render();
this.setEditMode(EditModes.SELECT);
},
});
this.commandManager.createNode(imageNode);
}
public addPaintNode(
canvas: HTMLCanvasElement,
lineBounds: {
width: number;
height: number;
x: number;
y: number;
},
) {
var node = new PaintNode({
canvasElement: canvas,
lineBounds: lineBounds,
mediaLayerRef: this.mediaLayer,
selectionManagerRef: this.selectionManager,
loaded: async () => {
await this.realTimeDrawEngine.render();
},
});
this.commandManager.createNode(node);
}
// Events for Undo and Redo
private addKeyboardShortcuts() {
window.addEventListener("keydown", (event) => {
if (event.ctrlKey && event.key === "z") {
this.undoStackManager.undo();
} else if (
(event.ctrlKey && event.key === "y") ||
(event.ctrlKey && event.shiftKey && event.key === "Z")
) {
this.undoStackManager.redo();
} else if (event.key === "Delete") {
this.commandManager.deleteNodes();
}
});
}
}
@@ -0,0 +1,7 @@
const ensureBase64Prefix = (base64String: string) => {
return base64String.startsWith("data:")
? base64String
: `data:image/png;base64,${base64String}`;
}
export { ensureBase64Prefix };
@@ -0,0 +1,46 @@
import { invoke } from "@tauri-apps/api/core";
import {
cancelGeneration,
finishGeneration,
onStartGeneration,
} from "~/signals/generation/generationSignals";
export class GenerationEngine {
private isGenerating = false;
constructor() {
console.log("GenerationEngine constructor");
this.attachEventListeners();
}
private attachEventListeners() {
onStartGeneration((prompt) => {
this.invokeGeneration(prompt);
});
}
private async invokeGeneration(prompt: string) {
if (this.isGenerating) {
return;
}
this.isGenerating = true;
console.log("Invoking generation for prompt:", prompt);
try {
const base64BitmapResponse = await invoke("text_to_image", {
prompt: prompt,
});
console.log("Generation finished with response:", base64BitmapResponse);
const base64string = base64BitmapResponse as string;
finishGeneration(base64string);
} catch (error) {
console.error("Error during image processing:", error);
cancelGeneration();
} finally {
this.isGenerating = false;
}
}
}
@@ -25,6 +25,7 @@ import {
isLoadingVisible,
loadingProgress,
} from "~/signals/uiEvents/loadingIndicator";
import { ensureBase64Prefix } from "../EngineUtitlities/Base64Helpers";
import {DecodeBase64ToImage} from "~/utilities/DecodeBase64ToImage.ts";
interface ServerSettings {
@@ -7,6 +7,11 @@ export enum AppModes {
RENDERING = "RENDERING",
}
export enum EditModes {
SELECT = "SELECT",
EDIT = "EDIT",
}
export const VideoResolutions = {
VERTICAL_720: { width: 720, height: 1280 },
LANDSCAPE_720: { width: 1280, height: 720 },
+7
View File
@@ -1,4 +1,5 @@
// to do fix https://codesandbox.io/p/sandbox/react-konva-infinite-grid-kkndq?file=%2Fsrc%2Findex.js the dotted background doesn't move when draggable.
import { EditEngine } from "./EditEngine";
import { Engine } from "./Engine";
import { EngineOptions } from "./types";
export { Engine as EngineType };
@@ -8,3 +9,9 @@ export const KonvaApp = (element: HTMLDivElement, options: EngineOptions) => {
engine.initializeStage(options.sceneToken);
return engine;
};
export const KonvaEditModeApp = (element: HTMLDivElement) => {
const engine = new EditEngine(element);
engine.initializeStage();
return engine;
};
+40 -4
View File
@@ -10,6 +10,13 @@ import { KonvaRootComponent } from "~/KonvaRootComponent";
import { ToolbarTopLeft } from "~/components/features/ToolbarTopLeft";
import { ToolbarTopRight } from "~/components/features/ToolbarTopRight";
import { ToolbarTopCenter } from "~/components/features/ToolbarTopCenter";
import { appMode } from "~/signals";
import { useSignals } from "@preact/signals-react/runtime";
import { GenerationRootComponent } from "~/GenerationRootComponent/GenerationRootComponent";
import { EditModeRootComponent } from "~/EditModeRootComponent/EditModeRootComponent";
import { useRef } from "react";
import { EngineType } from "~/KonvaApp";
import { GenerationEngine } from "~/KonvaApp/GenerationEngine";
export const Main = withProtectionRoute(() => {
// This is a hook that will log the number of times the component has rerendered
@@ -19,12 +26,41 @@ export const Main = withProtectionRoute(() => {
if (import.meta.env.DEV && sceneToken === "debug") {
sceneToken = "m_p8nkry6m5j22w586xyex0w4a4pznbx";
}
useSignals();
const realtimeEngineRef = useRef<EngineType | null>(null);
const generationEngineRef = useRef<GenerationEngine | null>(null);
const appModeValue = appMode.value;
let childView;
switch (appModeValue) {
case "edit":
childView = (
<EditModeRootComponent className="col-span-12 col-start-1 row-span-12 row-start-1" />
);
break;
case "generate":
childView = (
<GenerationRootComponent generationEngineRef={generationEngineRef} />
);
break;
case "realtime":
default:
childView = (
<KonvaRootComponent
sceneToken={sceneToken}
className="col-span-12 col-start-1 row-span-12 row-start-1"
/>
);
break;
}
return (
<div className="fixed grid h-full w-full grid-cols-12 grid-rows-12">
<KonvaRootComponent
sceneToken={sceneToken}
className="col-span-12 col-start-1 row-span-12 row-start-1"
/>
{childView}
<div className="absolute top-0 w-full p-3.5">
<div className="relative flex w-full">
<div className="absolute left-0">
@@ -1,7 +1,12 @@
import { twMerge } from "tailwind-merge";
import { TabSelector, TabItem } from "~/components/ui/TabSelector";
import { useComputed } from "@preact/signals-react";
import { appMode, dispatchUiEvents } from "~/signals";
import {
appMode,
dispatchUiEvents,
GenerationLoadingState,
generationSignal,
} from "~/signals";
// import { paperWrapperStyles } from "~/components/styles";
// import { faPlus, faQuestion } from "@fortawesome/pro-solid-svg-icons";
@@ -12,21 +17,26 @@ export const ToolbarTopLeft = () => {
const activeTab = useComputed(() => appMode.value);
const tabs: TabItem[] = [
{ id: "image", label: "Image" },
{ id: "realtime", label: "Realtime" },
{ id: "edit", label: "Edit" },
{ id: "generate", label: "Generate" },
// { id: "video", label: "Video" },
];
const handleTabChange = (tabId: string) => {
// Update the global signal
dispatchUiEvents.changeAppMode(tabId as "image" | "edit");
dispatchUiEvents.changeAppMode(tabId as "realtime" | "edit" | "generate");
};
const generationState = generationSignal.value;
console.log(appMode.value);
return (
<div
className={twMerge("z-20 flex h-fit w-fit items-center gap-8 pl-1 pr-4")}
className={twMerge(
"relative z-50 flex h-fit w-fit items-center gap-8 pl-1 pr-4",
)}
>
<img
src="/brand/mira-logo.png"
@@ -38,6 +48,9 @@ export const ToolbarTopLeft = () => {
tabs={tabs}
activeTab={activeTab.value}
onTabChange={handleTabChange}
disabled={
generationState.loadingState === GenerationLoadingState.GENERATING
}
/>
{/* <ToolbarButton icon={faPlus}>
@@ -22,7 +22,9 @@ export const ToolbarTopRight = () => {
const [aspectRatio, setAspectRatio] = useState("1:1");
return (
<div className={twMerge("z-20 flex h-fit w-fit items-center gap-2")}>
<div
className={twMerge("relative z-50 flex h-fit w-fit items-center gap-2")}
>
{import.meta.env.DEV && ( // only show in dev mode for now
<ToggleButton
label="Ratio"
@@ -1,6 +1,7 @@
import React, { useRef, useEffect, useState } from "react";
import { Tab, TabGroup, TabList } from "@headlessui/react";
import { twMerge } from "tailwind-merge";
import { Tooltip } from "./Tooltip";
export interface TabItem {
id: string;
@@ -12,6 +13,8 @@ interface TabSelectorProps {
activeTab: string;
onTabChange: (tabId: string) => void;
className?: string;
disabled?: boolean;
disabledMessage?: string;
}
export const TabSelector: React.FC<TabSelectorProps> = ({
@@ -19,6 +22,8 @@ export const TabSelector: React.FC<TabSelectorProps> = ({
activeTab,
onTabChange,
className,
disabled,
disabledMessage,
}) => {
// Find the index of the active tab
const selectedIndex = tabs.findIndex((tab) => tab.id === activeTab);
@@ -46,13 +51,17 @@ export const TabSelector: React.FC<TabSelectorProps> = ({
}
}, [selectedIndex, tabs]);
return (
const TabGroupElement = (
<TabGroup
selectedIndex={selectedIndex}
onChange={handleTabChange}
className={twMerge("w-full", className)}
className={twMerge(
"w-full",
className,
disabled ? "cursor-not-allowed opacity-60" : "",
)}
>
<TabList className="glass relative inline-flex min-w-fit overflow-x-auto rounded-lg p-1 shadow-lg">
<TabList className="glass relative inline-flex min-w-fit overflow-x-auto rounded-lg p-0.5 py-1 shadow-lg">
{/* Animated indicator */}
<div
className="absolute top-1 z-10 h-[calc(100%-8px)] rounded-md border-2 border-primary bg-primary/30 transition-all duration-200 ease-in-out"
@@ -66,10 +75,12 @@ export const TabSelector: React.FC<TabSelectorProps> = ({
<Tab
key={tab.id}
ref={(el) => (tabsRef.current[index] = el)}
disabled={disabled}
className={({ selected }) =>
twMerge(
"relative z-20 mx-0.5 min-w-max rounded-md border-2 border-transparent px-4 py-0.5 text-center font-semibold transition-all duration-200 ease-in-out",
selected ? "text-white" : "text-gray-300 hover:text-white",
disabled ? "cursor-not-allowed opacity-60" : "",
)
}
>
@@ -79,4 +90,18 @@ export const TabSelector: React.FC<TabSelectorProps> = ({
</TabList>
</TabGroup>
);
return (
<>
{disabled && (
<Tooltip
tip={disabledMessage ?? "Cannot change tab. Generation in progress."}
position="bottom"
>
{TabGroupElement}
</Tooltip>
)}
{!disabled && TabGroupElement}
</>
);
};
@@ -0,0 +1,50 @@
import { effect, signal } from "@preact/signals-react";
export enum GenerationLoadingState {
INIT,
GENERATING,
GENERATED,
}
const initValue = { loadingState: GenerationLoadingState.INIT, prompt: "" };
export const generationSignal = signal<{
loadingState: GenerationLoadingState;
prompt: string;
imageB64?: string;
}>(initValue);
export const startGeneration = (prompt: string) => {
generationSignal.value = {
loadingState: GenerationLoadingState.GENERATING,
prompt,
imageB64: undefined,
};
};
export const onStartGeneration = (callback: (prompt: string) => void) => {
effect(() => {
// If value just changed to GENERATING, notify the listeners
if (
generationSignal.value.loadingState === GenerationLoadingState.GENERATING
) {
callback(generationSignal.value.prompt);
}
});
};
export const finishGeneration = (imageB64: string) => {
generationSignal.value = {
loadingState: GenerationLoadingState.GENERATED,
prompt: generationSignal.value.prompt,
imageB64,
};
};
export const cancelGeneration = () => {
if (
generationSignal.value.loadingState === GenerationLoadingState.GENERATING
) {
generationSignal.value = initValue;
}
};
@@ -0,0 +1 @@
export * from "./generationSignals";
+1
View File
@@ -1,3 +1,4 @@
export * from "./authentication";
export * from "./uiAccess";
export * from "./uiEvents";
export * from "./generation";
@@ -3,10 +3,10 @@ import { signal, effect } from "@preact/signals-core";
// this is to control the app mode - BFlat
// Define the app mode types
export type AppModeType = "image" | "edit";
export type AppModeType = "realtime" | "edit" | "generate";
// Create a signal for the current app mode
export const appMode = signal<AppModeType>("image");
export const appMode = signal<AppModeType>("realtime");
// Create a function to change the app mode
export const changeAppMode = (mode: AppModeType) => {
+14 -6
View File
@@ -200,6 +200,14 @@
"@babel/helper-validator-identifier" "^7.24.7"
to-fast-properties "^2.0.0"
"@devnomic/marquee@^1.0.3":
version "1.0.3"
resolved "https://registry.npmjs.org/@devnomic/marquee/-/marquee-1.0.3.tgz"
integrity sha512-q8QUtNO8ff+UvfrKJsHBtxh4fbOfhGk6SSdK0Uu/XZFdEzrmiGJ/l0FuVHRYsPAmzgnhVVnz1dGNB5CWGLMVvw==
dependencies:
clsx "^2.1.0"
tailwind-merge "^2.2.0"
"@emotion/is-prop-valid@^1.1.0":
version "1.3.0"
resolved "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.3.0.tgz"
@@ -905,14 +913,14 @@
resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz"
integrity sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==
"@types/react-dom@^18.3.0":
"@types/react-dom@^18.2.18", "@types/react-dom@^18.3.0":
version "18.3.0"
resolved "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.0.tgz"
integrity sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg==
dependencies:
"@types/react" "*"
"@types/react@*", "@types/react@^18.3.3":
"@types/react@*", "@types/react@^18.2.47", "@types/react@^18.3.3":
version "18.3.4"
resolved "https://registry.npmjs.org/@types/react/-/react-18.3.4.tgz"
integrity sha512-J7W30FTdfCxDDjmfRM+/JqLHBIyl7xUIp9kwK637FGmY7+mkSFSe6L4jpZzhj5QMfLssSDP4/i75AKkrdC7/Jw==
@@ -1411,7 +1419,7 @@ cli-columns@^4.0.0:
string-width "^4.2.3"
strip-ansi "^6.0.1"
clsx@^2.0.0:
clsx@^2.0.0, clsx@^2.1.0:
version "2.1.1"
resolved "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz"
integrity sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==
@@ -3635,7 +3643,7 @@ react-colorful@^5.6.1:
resolved "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz"
integrity sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==
"react-dom@^16.8.0 || ^17.0.0 || ^18.0.0", react-dom@^18, react-dom@^18.0.0, "react-dom@^18.0.0 || ^19.0.0 || ^19.0.0-rc", react-dom@^18.3.1, "react-dom@>= 16.8.0", react-dom@>=16.4.0, react-dom@>=16.8, react-dom@>=16.8.0:
"react-dom@^16.8.0 || ^17.0.0 || ^18.0.0", react-dom@^18, react-dom@^18.0.0, "react-dom@^18.0.0 || ^19.0.0 || ^19.0.0-rc", react-dom@^18.2.0, react-dom@^18.3.1, "react-dom@>= 16.8.0", react-dom@>=16.4.0, react-dom@>=16.8, react-dom@>=16.8.0:
version "18.3.1"
resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz"
integrity sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==
@@ -3684,7 +3692,7 @@ react-router@6.26.1:
dependencies:
"@remix-run/router" "1.19.1"
"react@^16.14.0 || 17.x || 18.x", "react@^16.8.0 || ^17.0.0 || ^18.0.0", "react@^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0", react@^18, react@^18.0.0, "react@^18.0.0 || ^19.0.0 || ^19.0.0-rc", react@^18.3.1, "react@>= 16.8.0", react@>=16.3, react@>=16.4.0, react@>=16.8, react@>=16.8.0:
"react@^16.14.0 || 17.x || 18.x", "react@^16.8.0 || ^17.0.0 || ^18.0.0", "react@^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0", react@^18, react@^18.0.0, "react@^18.0.0 || ^19.0.0 || ^19.0.0-rc", react@^18.2.0, react@^18.3.1, "react@>= 16.8.0", react@>=16.3, react@>=16.4.0, react@>=16.8, react@>=16.8.0:
version "18.3.1"
resolved "https://registry.npmjs.org/react/-/react-18.3.1.tgz"
integrity sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==
@@ -4180,7 +4188,7 @@ tabbable@^6.0.0:
resolved "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz"
integrity sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==
tailwind-merge@^2.5.2:
tailwind-merge@^2.2.0, tailwind-merge@^2.5.2:
version "2.5.2"
resolved "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.2.tgz"
integrity sha512-kjEBm+pvD+6eAwzJL2Bi+02/9LFLal1Gs61+QB7HvTfQQ0aXwC5LGT8PEt1gS0CWKktKe6ysPTAy3cBC5MeiIg==