Compare commits

...
Author SHA1 Message Date
bflatastic 6e4b814d31 text to image tab (commented out) 2025-05-24 10:56:50 +07:00
10 changed files with 601 additions and 9 deletions
@@ -6,6 +6,7 @@ import {
faCube,
faFilm,
faPaintbrush,
faImage,
} from "@fortawesome/pro-solid-svg-icons";
import { useLocation, useParams } from "react-router-dom";
import { Button, ButtonLink } from "~/components";
@@ -51,7 +52,16 @@ const appMenuTabs: MenuIconItem[] = [
icon: <FontAwesomeIcon icon={faPaintbrush} />,
},
{ id: "3D", label: "3D Editor", icon: <FontAwesomeIcon icon={faCube} /> },
{ id: "VIDEO", label: "Video", icon: <FontAwesomeIcon icon={faFilm} /> },
{
id: "VIDEO",
label: "Prompt to Video",
icon: <FontAwesomeIcon icon={faFilm} />,
},
// {
// id: "IMAGE",
// label: "Prompt to Image",
// icon: <FontAwesomeIcon icon={faImage} />,
// },
];
export const topNavMediaId = signal<string>("");
@@ -116,6 +126,10 @@ export const TopBar = ({
console.log("Adding to Video scene");
topNavMediaId.value = media_id ?? "";
topNavMediaUrl.value = url;
} else if (currentAppTabId === "IMAGE") {
console.log("Adding to Image scene");
topNavMediaId.value = media_id ?? "";
topNavMediaUrl.value = url;
} else {
console.warn(`Unknown tab type: ${currentAppTabId}`);
}
@@ -160,7 +174,13 @@ export const TopBar = ({
{currentAppTabId === "3D" ? (
<SceneTitleInput pageName={pageName} />
) : (
<h1>{currentAppTabId === "2D" ? "Canvas" : "Generate Video"}</h1>
<h1>
{currentAppTabId === "2D"
? "Canvas"
: currentAppTabId === "VIDEO"
? "Generate Video"
: "Generate Image"}
</h1>
)}
</div>
@@ -9,6 +9,7 @@ import { PreviewFrameImage } from "./comps/PreviewFrameImage";
import { pageHeight, pageWidth } from "~/signals";
import { PoseModeSelector } from "./comps/PoseModeSelector";
import ImageToVideo from "../PageVideo/ImageToVideo";
import TextToImage from "../PageImage/TextToImage";
import {
timelineHeight,
sidePanelWidth,
@@ -312,6 +313,7 @@ export const PageEditor = () => {
const display3d = appTabId.value === "3D" ? "block" : "none";
const display2d = appTabId.value === "2D" ? "block" : "none";
const displayVideo = appTabId.value === "VIDEO" ? "block" : "none";
const displayImage = appTabId.value === "IMAGE" ? "block" : "none";
return (
<div className="w-screen">
@@ -423,7 +425,13 @@ export const PageEditor = () => {
<div style={{ display: displayVideo }}>
<ImageToVideo
className="h-full w-full"
imageMediaId={topNavMediaId.value}
imageUrl={topNavMediaUrl.value}
/>
</div>
<div style={{ display: displayImage }}>
<TextToImage
imageMediaId={topNavMediaId.value}
imageUrl={topNavMediaUrl.value}
/>
@@ -0,0 +1,18 @@
.tile-background-animation {
background-image: url("/resources/images/tilebg.png");
background-repeat: repeat;
background-size: 512px;
animation: background-animation-scrolling 60s linear infinite;
-webkit-animation: background-animation-scrolling 60s linear infinite;
filter: grayscale(100%);
-webkit-filter: grayscale(100%);
}
@keyframes background-animation-scrolling {
0% {
background-position: 0 0;
}
100% {
background-position: 512px 512px;
}
}
@@ -0,0 +1,16 @@
import "./Background.css";
export default function GalleryBackground() {
return (
<>
<div className="pointer-events-none fixed inset-0 z-[1] overflow-hidden bg-[radial-gradient(50%_50%_at_50%_50%,rgba(0,0,0,0.00)_49%,rgba(0,0,0,0.60)_100%)]" />
<div className="tile-background-animation fixed inset-0 z-0 overflow-hidden">
<div
className={`h-full w-full opacity-30 transition-opacity duration-1000`}
>
<div className="absolute inset-0 grid grid-cols-5 gap-6 px-6 opacity-35" />
</div>
</div>
</>
);
}
@@ -0,0 +1,99 @@
import { useState, useRef } from "react";
import { JobContextType } from "libs/common/src/lib/types";
import { PromptBoxImage } from "@storyteller/ui-promptbox";
import BackgroundGallery from "./BackgroundGallery";
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
import { faFilm, faClock } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
interface TextToImageProps {
imageMediaId?: string;
imageUrl?: string;
}
const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
const containerRef = useRef<HTMLDivElement>(null);
const [selectedModel, setSelectedModel] = useState<string>("Kling 2.0");
const [useSystemPrompt, setUseSystemPrompt] = useState(false);
const [isImageBeingProcessed, setIsImageBeingProcessed] = useState(false);
const [selectedImage, setSelectedImage] = useState<File | null>(null);
const [imagePreviewUrl, setImagePreviewUrl] = useState<string>("");
const [isDialogOpen, setIsDialogOpen] = useState(false);
const jobContext: JobContextType = {
jobTokens: [],
addJobToken: () => {},
removeJobToken: () => {},
clearJobTokens: () => {},
};
const modelList: PopoverItem[] = [
{
label: "Kling 2.0",
icon: <FontAwesomeIcon icon={faFilm} className="h-4 w-4" />,
selected: selectedModel === "Kling 2.0",
description: "High quality model",
badges: [{ label: "2 min.", icon: <FontAwesomeIcon icon={faClock} /> }],
},
{
label: "Runway",
icon: <FontAwesomeIcon icon={faFilm} className="h-4 w-4" />,
selected: selectedModel === "Runway",
description: "Fast and high-quality model",
badges: [{ label: "2 min.", icon: <FontAwesomeIcon icon={faClock} /> }],
},
];
const handleModelSelect = (item: PopoverItem) => {
setSelectedModel(item.label);
};
return (
<div
ref={containerRef}
className="flex h-[calc(100vh-56px)] w-full rounded-lg bg-[#121212]"
>
<div className="h-full w-full p-4">
<div className="flex h-full w-full flex-col items-center justify-center rounded-md pb-12">
<div className="relative z-20 mb-12 flex flex-col items-center justify-center text-center drop-shadow-xl">
<span className="text-7xl font-bold">Generate Image</span>
<span className="pt-2 text-xl opacity-80">
Add a prompt, then generate
</span>
</div>
<PromptBoxImage
useJobContext={() => {
// TODO: Implement job context logic
console.log("Using job context");
return jobContext;
}}
model={selectedModel}
imageMediaId={imageMediaId}
url={imageUrl ?? imagePreviewUrl ?? undefined}
onEnqueuePressed={async () => {
console.log("Enqueue pressed");
}}
/>
<BackgroundGallery />
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-2">
<PopoverMenu
items={modelList}
onSelect={handleModelSelect}
mode="hoverSelect"
panelTitle="Select Model"
panelClassName="min-w-[280px]"
buttonClassName="bg-transparent p-0 text-lg hover:bg-transparent text-white/80 hover:text-white"
showIconsInList
triggerLabel="Model"
/>
</div>
</div>
</div>
</div>
);
};
export default TextToImage;
@@ -4,12 +4,11 @@ export default function GalleryBackground() {
return (
<>
<div className="pointer-events-none fixed inset-0 z-[1] overflow-hidden bg-[radial-gradient(50%_50%_at_50%_50%,rgba(0,0,0,0.00)_49%,rgba(0,0,0,0.60)_100%)]" />
<div className="fixed inset-0 z-0 overflow-hidden tile-background-animation">
<div className="tile-background-animation fixed inset-0 z-0 overflow-hidden">
<div
className={`h-full w-full transition-opacity duration-1000 opacity-30`}
className={`h-full w-full opacity-30 transition-opacity duration-1000`}
>
<div className="absolute inset-0 grid grid-cols-5 gap-6 px-6 opacity-35">
</div>
<div className="absolute inset-0 grid grid-cols-5 gap-6 px-6 opacity-35" />
</div>
</div>
</>
@@ -4,7 +4,7 @@ import { signal } from "@preact/signals-react";
export const appTabId = signal("VIDEO");
export const setAppTabId = (newId: string) => {
if (newId != "2D" && newId != "3D" && newId != "VIDEO") {
if (newId != "2D" && newId != "3D" && newId != "VIDEO" && newId != "IMAGE") {
console.error("Provided app ID is not valid");
return;
}
@@ -1,3 +1,4 @@
export * from "./lib/PromptBox2D";
export * from "./lib/PromptBox3D";
export * from "./lib/PromptBoxVideo";
export * from "./lib/PromptBoxImage";
@@ -0,0 +1,432 @@
import { useState, useRef, useEffect } from "react";
import { useSignals } from "@preact/signals-react/runtime";
import { toast } from "@storyteller/ui-toaster";
import { JobContextType } from "@storyteller/common";
import { downloadFileFromUrl } from "@storyteller/api";
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { Button, ToggleButton } from "@storyteller/ui-button";
import { Modal } from "@storyteller/ui-modal";
import { FalKlingImageToVideo } from "@storyteller/tauri-api";
import {
faMessageXmark,
faMessageCheck,
faSparkles,
faSpinnerThird,
faTimes,
faPlus,
faImages,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangle,
faSquare,
faRectangleVertical,
} from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { IsDesktopApp } from "@storyteller/tauri-utils";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import {
CommandSuccessStatus,
GetAppPreferences,
} from "@storyteller/tauri-api";
import { SoundRegistry } from "@storyteller/soundboard";
interface ReferenceImage {
id: string;
url: string;
file: File;
mediaToken: string;
}
interface PromptBoxImageProps {
useJobContext: () => JobContextType;
onEnqueuePressed?: () => void | Promise<void>;
model: string;
imageMediaId?: string;
url?: string;
}
export const PromptBoxImage = ({
useJobContext,
onEnqueuePressed,
model,
imageMediaId,
url,
}: PromptBoxImageProps) => {
useSignals();
// for the image media id and url, we need to set the reference image gallery panel.
useEffect(() => {
if (imageMediaId && url) {
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: url,
file: new File([], "library-image"),
mediaToken: imageMediaId,
};
setReferenceImages([referenceImage]);
}
}, [imageMediaId, url]);
console.log("Is this a desktop app?", IsDesktopApp());
const [isModalOpen, setIsModalOpen] = useState(false);
const [content, setContent] = useState("");
const { jobTokens, addJobToken, removeJobToken, clearJobTokens } =
useJobContext();
const [prompt, setPrompt] = useState("");
const [isEnqueueing, setIsEnqueueing] = useState(false);
const [useSystemPrompt, setUseSystemPrompt] = useState(true);
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[]
);
const [referenceImages, setReferenceImages] = useState<ReferenceImage[]>([]);
const [uploadingImages, setUploadingImages] = useState<
{ id: string; file: File }[]
>([]);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "3:2",
selected: true,
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "2:3",
selected: false,
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "1:1",
selected: false,
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
]);
const fileInputRef = useRef<HTMLInputElement>(null);
const textareaRef = useRef<HTMLTextAreaElement>(null);
useEffect(() => {
if (textareaRef.current) {
textareaRef.current.style.height = "auto";
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
}
});
useEffect(() => {
if (imageMediaId && url) {
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: url,
file: new File([], "library-image"),
mediaToken: imageMediaId,
};
setReferenceImages([referenceImage]);
}
}, [imageMediaId, url]);
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
}))
);
};
const handleRemoveReference = (id: string) => {
setReferenceImages((prev) => prev.filter((img) => img.id !== id));
if (fileInputRef.current) {
fileInputRef.current.value = "";
}
};
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
const handleGallerySelect = () => setIsGalleryModalOpen(true);
const handleGalleryClose = () => {
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
};
const handleImageSelect = (id: string) => {
setSelectedGalleryImages((prev) => {
if (prev.includes(id)) {
return [];
}
return [id];
});
};
const handleGalleryImages = (selectedItems: GalleryItem[]) => {
// Clear existing reference images first
setReferenceImages([]);
// Only take the first selected item
const item = selectedItems[0];
if (!item || !item.fullImage) return;
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: item.fullImage,
file: new File([], "library-image"),
mediaToken: item.id,
};
setReferenceImages([referenceImage]);
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
};
const handleAction = (action: string) => {
switch (action) {
case "gallery":
handleGallerySelect();
break;
default:
console.log("Unknown action:", action);
}
};
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
e.preventDefault();
const pastedText = e.clipboardData.getData("text").trim();
setPrompt(pastedText);
};
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
setPrompt(e.target.value);
};
const handleEnqueue = async () => {
setIsEnqueueing(true);
console.log("generateResponse", generateResponse);
const generateResponse = await FalKlingImageToVideo({
image_media_token: referenceImages[0].mediaToken,
});
onEnqueuePressed?.();
if (generateResponse.status === CommandSuccessStatus.Success) {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image added to queue");
} else {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
}
// let errorMessage = "Failed to enqueue image";
// if ("error_type" in generateResponse) {
// switch (generateResponse.error_type) {
// case SoraImageRemixErrorType.SoraLoginRequired:
// errorMessage =
// "You need to log into Sora to continue. See the settings menu.";
// break;
// case SoraImageRemixErrorType.ServerError:
// errorMessage = "Server error. Failed to enqueue image.";
// break;
// case SoraImageRemixErrorType.TooManyConcurrentTasks:
// errorMessage =
// "You have too many Sora image generation tasks running. Please wait a moment.";
// break;
// case SoraImageRemixErrorType.SoraUsernameNotYetCreated:
// errorMessage =
// "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
// break;
// case SoraImageRemixErrorType.SoraIsHavingProblems:
// errorMessage = "Sora is having problems. Please try again later.";
// break;
// }
// }
// toast.error(errorMessage);
// }
setIsEnqueueing(false);
};
const getCurrentResolutionIcon = () => {
const selected = aspectRatioList.find((item) => item.selected);
if (!selected || !selected.icon) return faRectangle;
const iconElement = selected.icon as React.ReactElement;
return iconElement.props.icon;
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleEnqueue();
}
};
return (
<>
<Modal
isOpen={isModalOpen}
onClose={() => {
setIsModalOpen(false);
setContent("");
}}
>
{content}
</Modal>
<div className="relative z-20 flex flex-col gap-3">
{(referenceImages.length > 0 || uploadingImages.length > 0) && (
<div className="flex w-full gap-2">
{referenceImages.map((image) => (
<div
key={image.id}
className="glass relative aspect-square w-20 rounded-lg"
>
<img
src={image.url}
alt="Reference"
className="h-full w-full rounded-lg object-cover"
/>
<button
onClick={() => handleRemoveReference(image.id)}
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/40 text-white backdrop-blur-md transition-colors hover:bg-black"
>
<FontAwesomeIcon icon={faTimes} className="h-2.5 w-2.5" />
</button>
</div>
))}
{uploadingImages.map(({ id, file }) => {
const previewUrl = URL.createObjectURL(file);
return (
<div
key={id}
className="glass relative aspect-square w-20 overflow-hidden rounded-lg"
>
<div className="absolute inset-0">
<img
src={previewUrl}
alt="Uploading preview"
className="h-full w-full object-cover blur-sm"
/>
</div>
<div className="absolute inset-0 flex items-center justify-center bg-black/20">
<FontAwesomeIcon
icon={faSpinnerThird}
className="h-6 w-6 animate-spin text-white"
/>
</div>
</div>
);
})}
</div>
)}
<div className="glass w-[730px] rounded-xl p-4">
<div className="flex justify-center gap-2">
<PopoverMenu
mode="button"
panelTitle="Add Image"
items={[
{
label: "Choose from library",
selected: false,
icon: <FontAwesomeIcon icon={faImages} className="h-4 w-4" />,
action: "gallery",
},
]}
onPanelAction={handleAction}
showIconsInList
buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none"
triggerIcon={
<FontAwesomeIcon icon={faPlus} className="text-xl" />
}
/>
<textarea
ref={textareaRef}
rows={1}
placeholder="Choose the image from library (+) and describe what you want to happen in the video..."
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => {}}
onBlur={() => {}}
/>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<Tooltip
content="Aspect Ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentResolutionIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
<Tooltip
content={
useSystemPrompt
? "Use system prompt: ON"
: "Use system prompt: OFF"
}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={useSystemPrompt}
icon={faMessageXmark}
activeIcon={faMessageCheck}
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
/>
</Tooltip>
</div>
<div className="flex items-center gap-2">
<Button
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={!isEnqueueing ? faSparkles : undefined}
onClick={handleEnqueue}
disabled={isEnqueueing || !prompt.trim()}
>
{isEnqueueing ? (
<FontAwesomeIcon
icon={faSpinnerThird}
className="animate-spin text-lg"
/>
) : (
"Generate"
)}
</Button>
</div>
</div>
</div>
</div>
<GalleryModal
isOpen={!!isGalleryModalOpen}
onClose={handleGalleryClose}
mode="select"
selectedItemIds={selectedGalleryImages}
onSelectItem={handleImageSelect}
maxSelections={1}
onUseSelected={handleGalleryImages}
onDownloadClicked={downloadFileFromUrl}
/>
</>
);
};
@@ -69,7 +69,6 @@ export const PromptBoxVideo = ({
const [content, setContent] = useState("");
const { jobTokens, addJobToken, removeJobToken, clearJobTokens } =
useJobContext();
const [prompt, setPrompt] = useState("");
const [isEnqueueing, setIsEnqueueing] = useState(false);
const [useSystemPrompt, setUseSystemPrompt] = useState(true);