Reference decks + new generate button for image/video promptboxes (#1783)

Redesigns how references are attached in the image and video
promptboxes, on both the desktop app and the webapp (desktop layout -
mobile is unchanged).

### Reference deck
- Replaces the image-toggle icon and the upload bands above the prompt
with a fanned stack of reference cards inside the promptbox, left of the
prompt text
- Hovering the deck expands it to show every attached ref - no
scrolling, it widens toward the end of the promptbox and wraps into more
rows as needed (ready for up to 50 refs for Seedance 2.5)
- Cards support drag-to-reorder (grab/grabbing cursors), hover remove,
name tooltips (Image 1, Video 2, …), upload progress spinners on
individual cards, and click-to-preview fullscreen
- A circular **+** overlaps the deck: hovering shows an add menu grouped
by type (Image / Video / Audio) with compact limits (e.g. `2/9`, `1/3 ·
8/15s`); clicking it opens a file picker that accepts any supported
media and sorts files by type automatically

### Video modes
- Keyframe mode: Start/End frame band replaced with compact "First frame
⇄ Last frame" cards with a swap button
- Reference mode renamed to **Omni reference** and is now the default;
images, videos, and audio all stack into one mixed deck

### Generate button
- Replaced the "Generate" pill with a circular arrow-up button, credit
cost shown beside it

### Under the hood
- New shared deck components live in `@storyteller/ui-promptbox` and are
used by both apps (previously the two promptbox implementations were
fully separate)
- New `GenerateIconButton` in `@storyteller/ui-button`; the old
`GenerateButton` stays for mobile
- Webapp mobile, audio/object/world pages, and the 2D/3D/Edit
promptboxes keep their existing UI
This commit is contained in:
BFlat
2026-07-14 21:08:15 +07:00
committed by GitHub
parent dbe38a588f
commit 46d05808cc
29 changed files with 3063 additions and 418 deletions
@@ -401,11 +401,13 @@ export function Lightbox({
</video>
) : isAudio ? (
<div className="flex h-full w-full items-center justify-center px-4 sm:px-8">
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-white/[0.04] px-2 py-8">
<FontAwesomeIcon
icon={faMusic}
className="mx-auto mb-5 block text-3xl text-white/20"
/>
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-gradient-to-br from-white/[0.08] via-white/[0.04] to-white/[0.02] px-4 py-10 sm:px-6">
<div className="mx-auto mb-6 flex h-16 w-16 items-center justify-center rounded-full bg-white/10 ring-1 ring-white/15">
<FontAwesomeIcon
icon={faMusic}
className="text-2xl text-white/70"
/>
</div>
<WaveformAudioPlayer
key={selectedImageUrl}
src={selectedImageUrl}
@@ -32,6 +32,7 @@ interface MediaReferenceRowProps {
onReferenceAudiosChange: (audios: RefAudio[]) => void;
maxAudioCount: number;
maxAudioRefDuration: number;
onPickAudioFromLibrary?: () => void;
className?: string;
}
@@ -47,6 +48,7 @@ export const MediaReferenceRow = ({
onReferenceAudiosChange,
maxAudioCount,
maxAudioRefDuration,
onPickAudioFromLibrary,
className,
}: MediaReferenceRowProps) => {
const videoInputRef = useRef<HTMLInputElement>(null);
@@ -303,7 +305,11 @@ export const MediaReferenceRow = ({
</div>
)}
{canAddAudio && (
<AddButton onUpload={() => audioInputRef.current?.click()} />
<AddButton
onUpload={() => audioInputRef.current?.click()}
onPickFromLibrary={onPickAudioFromLibrary}
title="Add audio"
/>
)}
</div>
</div>
@@ -16,12 +16,21 @@ import {
faUserGroup,
faVideo,
} from "@fortawesome/pro-solid-svg-icons";
import { GenerateButton } from "@storyteller/ui-button";
import { GenerateIconButton } from "@storyteller/ui-button";
import { Tooltip } from "@storyteller/ui-tooltip";
import { ImagePromptRow } from "./ImagePromptRow";
import {
KeyframeCards,
ReferenceDeck,
useDeckMedia,
type DeckAddAction,
type DeckItem,
} from "@storyteller/ui-promptbox";
import { arrayMove } from "@dnd-kit/sortable";
import { MentionTextarea } from "./MentionTextarea";
import { getMentionColor, buildMentionColorMap } from "./mention-colors";
import type { RefImage, MentionItem } from "./types";
import { uploadImage } from "./upload-image";
import { uploadVideo, uploadAudio } from "./upload-media";
import type { RefImage, RefVideo, RefAudio, MentionItem } from "./types";
import { useEnterToGenerateStore } from "../../lib/enter-to-generate-store";
import {
PromptFullscreenButton,
@@ -36,7 +45,6 @@ interface PromptBoxProps {
onPromptChange: (prompt: string) => void;
onSubmit: () => void;
isSubmitting: boolean;
submitLabel?: string;
placeholder?: string;
disabled?: boolean;
credits?: number | null;
@@ -61,10 +69,25 @@ interface PromptBoxProps {
// Pick from library
onPickFromLibrary?: () => void;
onPickEndFrameFromLibrary?: () => void;
onPickVideoFromLibrary?: () => void;
onPickAudioFromLibrary?: () => void;
// Clear all references (images, end frame, videos, audios)
onClearAllRefs?: () => void;
// Media reference row (video/audio refs, rendered between image row and prompt)
// Reference-mode media (video page passes these; other pages omit)
videoRefsSupported?: boolean;
referenceVideos?: RefVideo[];
onReferenceVideosChange?: (videos: RefVideo[]) => void;
maxVideoCount?: number;
maxVideoRefDuration?: number;
audioRefsSupported?: boolean;
referenceAudios?: RefAudio[];
onReferenceAudiosChange?: (audios: RefAudio[]) => void;
maxAudioCount?: number;
maxAudioRefDuration?: number;
// Legacy band rendered above the box (audio/object/world pages still use
// this; the image/video pages moved to the inline reference deck).
mediaReferenceRow?: ReactNode;
// Model selector (rendered above the toolbar, typically hidden on desktop via lg:hidden)
@@ -90,7 +113,6 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
onPromptChange,
onSubmit,
isSubmitting,
submitLabel = "Generate",
placeholder = "Describe what you want...",
disabled,
credits,
@@ -107,7 +129,19 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
rightToolbar,
onPickFromLibrary,
onPickEndFrameFromLibrary,
onPickVideoFromLibrary,
onPickAudioFromLibrary,
onClearAllRefs,
videoRefsSupported,
referenceVideos = [],
onReferenceVideosChange,
maxVideoCount = 3,
maxVideoRefDuration = 30,
audioRefsSupported,
referenceAudios = [],
onReferenceAudiosChange,
maxAudioCount = 2,
maxAudioRefDuration = 30,
mediaReferenceRow,
modelSelector,
secondaryPromptRow,
@@ -122,7 +156,6 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
const enterToGenerate = useEnterToGenerateStore((s) => s.enabled);
const [isFocused, setIsFocused] = useState(false);
const [isExpanded, setIsExpanded] = useState(false);
const [showImagePrompts, setShowImagePrompts] = useState(false);
const { isFullscreen, openFullscreen, closeFullscreen } =
useFullscreenPrompt();
@@ -148,12 +181,298 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
const [mentionIndex, setMentionIndex] = useState(0);
const mentionAnchorRef = useRef<number | null>(null);
const isImageRowVisible =
supportsImagePrompts &&
(isVideo || showImagePrompts || referenceImages.length > 0);
const hasMentionItems = (mentionItems?.length ?? 0) > 0;
const hasAnyRowAbove = isImageRowVisible || !!mediaReferenceRow;
const deck = useDeckMedia({
referenceImages,
setReferenceImages: onReferenceImagesChange,
maxImages: maxImagePromptCount,
setEndFrameImage: onEndFrameImageChange,
referenceVideos,
setReferenceVideos: onReferenceVideosChange,
maxVideos: maxVideoCount,
maxVideoTotalSec: maxVideoRefDuration,
referenceAudios,
setReferenceAudios: onReferenceAudiosChange,
maxAudios: maxAudioCount,
maxAudioTotalSec: maxAudioRefDuration,
uploadImage,
uploadVideo,
uploadAudio,
// Library picking stays with the page-level GalleryModals.
ownGalleryModal: false,
});
// Mixed deck items ordered images → videos → audios so the page's
// index-derived @ImageN/@VideoN/@AudioN mention labels stay aligned.
const deckItems: DeckItem[] = useMemo(
() => [
...referenceImages.map((img, i) => ({
id: img.id,
kind: "image" as const,
url: img.url,
previewUrl: img.fullUrl ?? img.url,
name: `Image ${i + 1}`,
})),
...deck.uploadingImages.map((entry, i) => ({
id: entry.id,
kind: "image" as const,
url: entry.previewUrl,
name: `Image ${referenceImages.length + i + 1}`,
uploading: true,
})),
...referenceVideos.map((video, i) => ({
id: video.id,
kind: "video" as const,
url: video.url,
name: `Video ${i + 1}`,
duration: video.duration,
})),
...(deck.uploadingVideo
? [
{
id: deck.uploadingVideo.id,
kind: "video" as const,
url: deck.uploadingVideo.previewUrl,
name: `Video ${referenceVideos.length + 1}`,
uploading: true,
},
]
: []),
...referenceAudios.map((audio, i) => ({
id: audio.id,
kind: "audio" as const,
url: audio.url,
name: `Audio ${i + 1}`,
duration: audio.duration,
})),
...(deck.uploadingAudio
? [
{
id: deck.uploadingAudio.id,
kind: "audio" as const,
name: `Audio ${referenceAudios.length + 1}`,
uploading: true,
},
]
: []),
],
[
referenceImages,
referenceVideos,
referenceAudios,
deck.uploadingImages,
deck.uploadingVideo,
deck.uploadingAudio,
],
);
const deckAddActions: DeckAddAction[] = [];
if (
referenceImages.length + deck.uploadingImages.length <
maxImagePromptCount
) {
deckAddActions.push({
key: "upload-image",
label: "Upload",
group: "image",
onSelect: deck.openImageUpload,
});
if (onPickFromLibrary) {
deckAddActions.push({
key: "library-image",
label: isVideo ? "From library" : "Pick from library",
group: "image",
onSelect: onPickFromLibrary,
});
}
}
if (
isVideo &&
isReferenceMode &&
videoRefsSupported &&
referenceVideos.length < maxVideoCount &&
!deck.uploadingVideo
) {
deckAddActions.push({
key: "upload-video",
label: "Upload",
group: "video",
onSelect: deck.openVideoUpload,
});
if (onPickVideoFromLibrary) {
deckAddActions.push({
key: "library-video",
label: "From library",
group: "video",
onSelect: onPickVideoFromLibrary,
});
}
}
if (
isVideo &&
isReferenceMode &&
audioRefsSupported &&
referenceAudios.length < maxAudioCount &&
!deck.uploadingAudio
) {
deckAddActions.push({
key: "upload-audio",
label: "Upload",
group: "audio",
onSelect: deck.openAudioUpload,
});
if (onPickAudioFromLibrary) {
deckAddActions.push({
key: "library-audio",
label: "From library",
group: "audio",
onSelect: onPickAudioFromLibrary,
});
}
}
const handleRemoveDeckItem = (id: string) => {
if (referenceImages.some((img) => img.id === id)) {
onReferenceImagesChange(referenceImages.filter((img) => img.id !== id));
} else if (referenceVideos.some((video) => video.id === id)) {
onReferenceVideosChange?.(
referenceVideos.filter((video) => video.id !== id),
);
} else if (referenceAudios.some((audio) => audio.id === id)) {
onReferenceAudiosChange?.(
referenceAudios.filter((audio) => audio.id !== id),
);
}
};
const firstFrameItem: DeckItem | undefined = referenceImages[0]
? {
id: referenceImages[0].id,
kind: "image",
url: referenceImages[0].url,
previewUrl: referenceImages[0].fullUrl ?? referenceImages[0].url,
name: "First frame",
}
: deck.uploadingImages[0]
? {
id: deck.uploadingImages[0].id,
kind: "image",
url: deck.uploadingImages[0].previewUrl,
name: "First frame",
uploading: true,
}
: undefined;
const lastFrameItem: DeckItem | undefined = endFrameImage
? {
id: endFrameImage.id,
kind: "image",
url: endFrameImage.url,
previewUrl: endFrameImage.fullUrl ?? endFrameImage.url,
name: "Last frame",
}
: deck.uploadingEnd
? {
id: deck.uploadingEnd.id,
kind: "image",
url: deck.uploadingEnd.previewUrl,
name: "Last frame",
uploading: true,
}
: undefined;
const handleSwapFrames = () => {
const first = referenceImages[0];
if (!first || !endFrameImage) return;
onReferenceImagesChange([endFrameImage]);
onEndFrameImageChange?.(first);
};
// Left-of-textarea reference widget: image deck, keyframe cards, or the
// mixed reference-mode deck depending on page/mode.
const renderReferenceWidget = (alwaysExpanded?: boolean) => {
if (!supportsImagePrompts) return null;
if (isVideo && !isReferenceMode) {
return (
<KeyframeCards
firstFrame={firstFrameItem}
lastFrame={lastFrameItem}
showLastFrame={!!showEndFrameSection}
onFirstAddActions={[
{
key: "upload-first",
label: "Upload",
onSelect: deck.openImageUpload,
},
...(onPickFromLibrary
? [
{
key: "library-first",
label: "Pick from library",
onSelect: onPickFromLibrary,
},
]
: []),
]}
onLastAddActions={[
{
key: "upload-last",
label: "Upload",
onSelect: deck.openEndUpload,
},
...(onPickEndFrameFromLibrary
? [
{
key: "library-last",
label: "Pick from library",
onSelect: onPickEndFrameFromLibrary,
},
]
: []),
]}
onRemoveFirst={() => onReferenceImagesChange([])}
onRemoveLast={() => onEndFrameImageChange?.(undefined)}
onSwap={handleSwapFrames}
/>
);
}
const totalVideoRefSeconds = referenceVideos.reduce(
(sum, video) => sum + video.duration,
0,
);
const totalAudioRefSeconds = referenceAudios.reduce(
(sum, audio) => sum + audio.duration,
0,
);
const groupHints: Record<string, string> = {
image: `${referenceImages.length}/${maxImagePromptCount}`,
};
if (isVideo && isReferenceMode) {
if (videoRefsSupported) {
groupHints.video = `${referenceVideos.length}/${maxVideoCount} · ${totalVideoRefSeconds}/${maxVideoRefDuration}s`;
}
if (audioRefsSupported) {
groupHints.audio = `${referenceAudios.length}/${maxAudioCount} · ${totalAudioRefSeconds}/${maxAudioRefDuration}s`;
}
}
return (
<ReferenceDeck
items={deckItems}
canAdd={deckAddActions.length > 0}
addActions={deckAddActions}
addMenuGroupHints={groupHints}
onAddClick={deck.openAnyUpload}
onRemove={handleRemoveDeckItem}
onReorderImages={(from, to) =>
onReferenceImagesChange(arrayMove(referenceImages, from, to))
}
onClearAll={onClearAllRefs}
alwaysExpanded={alwaysExpanded}
/>
);
};
// Filtered mention items for autocomplete
const filteredMentionItems = useMemo(() => {
@@ -334,80 +653,24 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
});
}, [prompt, hasMentionItems, mentionRegex, mentionLabelMap, mentionItems]);
// Shared reference-image row. Inline it attaches to the top of the box
// (square bottom); the fullscreen modal passes a className to round the
// bottom corners since it sits in-flow above the controls there.
const renderImagePromptRow = (className?: string) =>
supportsImagePrompts ? (
<ImagePromptRow
maxImagePromptCount={maxImagePromptCount}
referenceImages={referenceImages}
setReferenceImages={onReferenceImagesChange}
onPickFromLibrary={onPickFromLibrary}
onClearAll={onClearAllRefs}
isVideo={isVideo}
isReferenceMode={isReferenceMode}
endFrameImage={endFrameImage}
setEndFrameImage={onEndFrameImageChange}
showEndFrameSection={showEndFrameSection}
onPickEndFrameFromLibrary={onPickEndFrameFromLibrary}
className={className}
/>
) : null;
return (
<div ref={ref} className="prompt-box-root">
<div className="relative flex flex-col">
{isImageRowVisible && renderImagePromptRow()}
{deck.fileInputs}
{/* When the media row is the topmost band (no image row above it —
e.g. the audio page), clip it to rounded top corners; mid-stack
(video's start-frame row above) it stays square. */}
{mediaReferenceRow && (
<div className={twMerge(!isImageRowVisible && "sm:rounded-t-2xl")}>
{mediaReferenceRow}
</div>
<div className="sm:rounded-t-2xl">{mediaReferenceRow}</div>
)}
<div
className={twMerge(
"glass rounded-2xl p-3 sm:p-4 !transition-all duration-200",
hasAnyRowAbove && "rounded-t-none border-t-0",
mediaReferenceRow && "rounded-t-none border-t-0",
isFocused && "ring-1 ring-primary",
)}
>
<div className="flex gap-3">
{supportsImagePrompts && !isVideo && (
<Tooltip
content="Add Image"
position="top"
closeOnClick={true}
className={twMerge(isImageRowVisible && "hidden opacity-0")}
>
<button
type="button"
className={twMerge(
"flex h-6 w-6 shrink-0 items-center justify-center rounded-md bg-transparent p-0 transition-all hover:text-primary-500",
isImageRowVisible && "text-primary",
)}
onClick={() => setShowImagePrompts((prev) => !prev)}
>
<svg
width="24"
height="20"
viewBox="0 0 24 20"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className="opacity-80 transition-all hover:opacity-100"
>
<path
d="M2.66667 2H16C16.3667 2 16.6667 2.3 16.6667 2.66667V6.1125C17.1 6.04167 17.5458 6 18 6C18.225 6 18.4458 6.00833 18.6667 6.02917V2.66667C18.6667 1.19583 17.4708 0 16 0H2.66667C1.19583 0 0 1.19583 0 2.66667V16C0 17.4708 1.19583 18.6667 2.66667 18.6667H11.5C11.0625 18.0583 10.7083 17.3875 10.4542 16.6667H2.66667C2.3 16.6667 2 16.3667 2 16V2.66667C2 2.3 2.3 2 2.66667 2ZM11.8625 7.49167C11.6833 7.1875 11.3542 7 11 7C10.6458 7 10.3167 7.1875 10.1375 7.49167L8.2 10.7833L7.48333 9.75833C7.29583 9.49167 6.99167 9.33333 6.6625 9.33333C6.33333 9.33333 6.02917 9.49167 5.84167 9.75833L3.50833 13.0917C3.29583 13.3958 3.26667 13.7958 3.44167 14.125C3.61667 14.4542 3.9625 14.6667 4.33333 14.6667H10.0292C10.0125 14.4458 10 14.225 10 14C10 11.7833 10.9 9.77917 12.3542 8.33333L11.8625 7.49583V7.49167ZM5.33333 6.66667C6.07083 6.66667 6.66667 6.07083 6.66667 5.33333C6.66667 4.59583 6.07083 4 5.33333 4C4.59583 4 4 4.59583 4 5.33333C4 6.07083 4.59583 6.66667 5.33333 6.66667ZM18 20C21.3125 20 24 17.3125 24 14C24 10.6875 21.3125 8 18 8C14.6875 8 12 10.6875 12 14C12 17.3125 14.6875 20 18 20ZM18.6667 11.3333V13.3333H20.6667C21.0333 13.3333 21.3333 13.6333 21.3333 14C21.3333 14.3667 21.0333 14.6667 20.6667 14.6667H18.6667V16.6667C18.6667 17.0333 18.3667 17.3333 18 17.3333C17.6333 17.3333 17.3333 17.0333 17.3333 16.6667V14.6667H15.3333C14.9667 14.6667 14.6667 14.3667 14.6667 14C14.6667 13.6333 14.9667 13.3333 15.3333 13.3333H17.3333V11.3333C17.3333 10.9667 17.6333 10.6667 18 10.6667C18.3667 10.6667 18.6667 10.9667 18.6667 11.3333Z"
fill="currentColor"
/>
</svg>
</button>
</Tooltip>
)}
{renderReferenceWidget()}
<div className="promptbox-resize-wrap relative flex-1">
{hasMentionItems && mentionItems ? (
@@ -561,22 +824,19 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
)}
{/* Toolbar */}
<div className="mt-2 flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
<div className="mt-3.5 flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
<div className="flex flex-wrap items-center gap-1.5 sm:gap-2">
{modelSelector}
{leftToolbar}
</div>
<div className="flex items-center gap-2 sm:shrink-0">
{rightToolbar}
<GenerateButton
className="flex flex-1 sm:flex-none items-center justify-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
<GenerateIconButton
onClick={onSubmit}
disabled={disabled ?? (!prompt.trim() || isSubmitting)}
loading={isSubmitting}
credits={credits}
>
{submitLabel}
</GenerateButton>
/>
</div>
</div>
@@ -611,7 +871,7 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
{leftToolbar}
</>
}
imagePromptRow={renderImagePromptRow("rounded-2xl sm:rounded-b-2xl")}
imagePromptRow={renderReferenceWidget(true) ?? undefined}
>
{hasMentionItems && mentionItems ? (
<MentionTextarea
@@ -1,10 +1,4 @@
import {
ReactNode,
useCallback,
useEffect,
useRef,
useState,
} from "react";
import { ReactNode, useCallback, useEffect, useRef, useState } from "react";
import { Modal } from "@storyteller/ui-modal";
import { Button } from "@storyteller/ui-button";
import { Tooltip } from "@storyteller/ui-tooltip";
@@ -86,7 +80,10 @@ export const PromptFullscreenModal = ({
// Focus the editor and drop the caret at the end once the overlay opens.
useEffect(() => {
if (!isOpen) return;
const id = window.setTimeout(() => focusEditorAtEnd(contentRef.current), 60);
const id = window.setTimeout(
() => focusEditorAtEnd(contentRef.current),
60,
);
return () => window.clearTimeout(id);
}, [isOpen]);
@@ -123,12 +120,13 @@ export const PromptFullscreenModal = ({
overLimit ? "text-red-500" : "text-base-fg/40"
}`}
>
{promptLength} / {isFinite(maxPromptLength) ? maxPromptLength : "∞"}
{promptLength} /{" "}
{isFinite(maxPromptLength) ? maxPromptLength : "∞"}
</span>
</div>
)}
{imagePromptRow && <div className="shrink-0">{imagePromptRow}</div>}
<div className="flex shrink-0 items-center justify-between gap-2">
<div className="flex shrink-0 items-center justify-between gap-2 mt-1">
<div className="flex items-center gap-2">{footerControls}</div>
<Button onClick={onClose}>Done</Button>
</div>
@@ -5,7 +5,10 @@ export { CharactersModal } from "./CharactersModal";
export { useCharactersStore } from "./characters-store";
export type { StoredCharacter } from "./characters-store";
export type { RefImage, RefVideo, RefAudio, MentionItem } from "./types";
export { getVideoDurationFromUrl } from "./upload-media";
export {
getAudioDurationFromUrl,
getVideoDurationFromUrl,
} from "./upload-media";
export {
MobilePromptForm,
MobileSelectField,
@@ -83,6 +83,16 @@ export const getVideoDurationFromUrl = (url: string): Promise<number> =>
video.src = url;
});
// Resolves 0 when metadata can't be loaded.
export const getAudioDurationFromUrl = (url: string): Promise<number> =>
new Promise((resolve) => {
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.onloadedmetadata = () => resolve(Math.round(audio.duration));
audio.onerror = () => resolve(0);
audio.src = url;
});
export const getVideoDuration = (file: File): Promise<number> => {
const url = URL.createObjectURL(file);
return getVideoDurationFromUrl(url).finally(() => URL.revokeObjectURL(url));
@@ -31,6 +31,8 @@ import {
SettingsDrawer,
DrawerOptionList,
DrawerSection,
getAudioDurationFromUrl,
type RefAudio,
type RefImage,
} from "../../components/prompt-box";
import {
@@ -190,6 +192,10 @@ export default function CreateAudio() {
const setReferenceImages = useCreateAudioStore((s) => s.setReferenceImages);
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
const [isAudioPickerOpen, setIsAudioPickerOpen] = useState(false);
const [audioPickerSelectedIds, setAudioPickerSelectedIds] = useState<
string[]
>([]);
const [isBeatDrawerOpen, setIsBeatDrawerOpen] = useState(false);
const [isTuningDrawerOpen, setIsTuningDrawerOpen] = useState(false);
@@ -197,6 +203,10 @@ export default function CreateAudio() {
if (isImagePickerOpen) setPickerSelectedIds([]);
}, [isImagePickerOpen]);
useEffect(() => {
if (isAudioPickerOpen) setAudioPickerSelectedIds([]);
}, [isAudioPickerOpen]);
const handlePickerSelect = useCallback(
(id: string) => {
setPickerSelectedIds((prev) => {
@@ -295,6 +305,62 @@ export default function CreateAudio() {
[referenceAudios, setReferenceAudios, setReferenceImages],
);
const audioPickerMax = Math.max(1, maxAudioRefs - referenceAudios.length);
const handleAudioPickerSelect = useCallback(
(id: string) => {
setAudioPickerSelectedIds((prev) => {
if (prev.includes(id)) return prev.filter((x) => x !== id);
if (prev.length >= audioPickerMax) {
return audioPickerMax === 1 ? [id] : prev;
}
return [...prev, id];
});
},
[audioPickerMax],
);
const handleLibraryAudioSelect = useCallback(
async (items: GalleryItem[]) => {
setIsAudioPickerOpen(false);
const availableSlots = Math.max(0, maxAudioRefs - referenceAudios.length);
const picked = items.slice(0, availableSlots);
const added: RefAudio[] = [];
let total = referenceAudios.reduce((sum, a) => sum + a.duration, 0);
for (const item of picked) {
const url = item.fullImage;
if (!url) continue;
const duration =
item.durationMillis != null
? Math.round(item.durationMillis / 1000)
: await getAudioDurationFromUrl(url);
if (duration <= 0) {
toast.error("Could not read audio file");
continue;
}
if (total + duration > AUDIO_REF_MAX_DURATION_SECONDS) {
toast.error(
`Total audio duration cannot exceed ${AUDIO_REF_MAX_DURATION_SECONDS}s`,
);
continue;
}
total += duration;
added.push({
id: Math.random().toString(36).substring(7),
url,
file: new File([], "library-audio"),
mediaToken: item.id,
duration,
});
}
if (added.length > 0) {
handleReferenceAudiosChange([...referenceAudios, ...added]);
}
},
[referenceAudios, maxAudioRefs, handleReferenceAudiosChange],
);
const handleLibraryImageSelect = useCallback(
(items: GalleryItem[]) => {
const newImages: RefImage[] = items.slice(0, maxImageRefs).map((item) => ({
@@ -493,6 +559,7 @@ export default function CreateAudio() {
onReferenceAudiosChange={handleReferenceAudiosChange}
maxAudioCount={maxAudioRefs}
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
onPickAudioFromLibrary={() => setIsAudioPickerOpen(true)}
/>
) : undefined;
@@ -798,6 +865,17 @@ export default function CreateAudio() {
forceFilter="image"
hideFilter
/>
<GalleryModal
mode="select"
isOpen={isAudioPickerOpen}
onClose={() => setIsAudioPickerOpen(false)}
selectedItemIds={audioPickerSelectedIds}
onSelectItem={handleAudioPickerSelect}
maxSelections={audioPickerMax}
onUseSelected={handleLibraryAudioSelect}
forceFilter="audio"
hideFilter
/>
<Lightbox
isOpen={lightbox.lightboxOpen}
onClose={lightbox.closeLightbox}
@@ -72,7 +72,7 @@ const DEFAULT_UI: VideoUiState = {
resolution: null,
bitrate: null,
generateWithSound: false,
inputMode: "keyframe",
inputMode: "reference",
numVideos: 1,
};
@@ -161,6 +161,21 @@ export const useCreateVideoStore = create<CreateVideoState>()(
}),
{
name: "artcraft-video-batches",
// Bumped to 1 when reference became the default input mode: the
// migration runs once for pre-existing persisted state and resets the
// stored mode so everyone lands on the new default.
version: 1,
migrate: (persisted, version) => {
const p = (persisted ?? {}) as {
batches?: VideoBatch[];
ui?: VideoUiState;
};
const ui = { ...DEFAULT_UI, ...(p.ui ?? {}) };
if (version < 1) {
ui.inputMode = "reference";
}
return { batches: p.batches ?? [], ui };
},
// Persist prompt + lightweight settings alongside pending batches so a
// full page reload (e.g. returning from a credit top-up) keeps the
// user's draft. Reference media (refs) is excluded for the same reason
@@ -27,6 +27,7 @@ import {
SettingsDrawer,
DrawerOptionList,
DrawerSection,
getAudioDurationFromUrl,
getVideoDurationFromUrl,
type RefImage,
type RefVideo,
@@ -367,6 +368,7 @@ export default function CreateVideo() {
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
const [isEndFramePickerOpen, setIsEndFramePickerOpen] = useState(false);
const [isVideoRefPickerOpen, setIsVideoRefPickerOpen] = useState(false);
const [isAudioRefPickerOpen, setIsAudioRefPickerOpen] = useState(false);
const [isCharactersModalOpen, setIsCharactersModalOpen] = useState(false);
const [isOutputDrawerOpen, setIsOutputDrawerOpen] = useState(false);
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
@@ -376,6 +378,9 @@ export default function CreateVideo() {
const [videoRefPickerSelectedIds, setVideoRefPickerSelectedIds] = useState<
string[]
>([]);
const [audioRefPickerSelectedIds, setAudioRefPickerSelectedIds] = useState<
string[]
>([]);
useEffect(() => {
if (isImagePickerOpen) setPickerSelectedIds([]);
@@ -389,6 +394,10 @@ export default function CreateVideo() {
if (isVideoRefPickerOpen) setVideoRefPickerSelectedIds([]);
}, [isVideoRefPickerOpen]);
useEffect(() => {
if (isAudioRefPickerOpen) setAudioRefPickerSelectedIds([]);
}, [isAudioRefPickerOpen]);
// Characters store for @-mentions
const storedCharacters = useCharactersStore((s) => s.characters);
const charactersLoaded = useCharactersStore((s) => s.loaded);
@@ -647,7 +656,7 @@ export default function CreateVideo() {
selected: inputMode === "keyframe",
},
{
label: "Reference",
label: "Omni Reference",
description: "Multi-media ref",
selected: inputMode === "reference",
},
@@ -802,7 +811,7 @@ export default function CreateVideo() {
const handleInputModeChange = useCallback(
(item: PopoverItem) => {
const mode = item.label === "Reference" ? "reference" : "keyframe";
const mode = item.label === "Omni Reference" ? "reference" : "keyframe";
if (mode === inputMode) return;
if (mode === "reference") {
// Some models (e.g. Grok) cap duration lower in image-reference mode.
@@ -923,6 +932,70 @@ export default function CreateVideo() {
[referenceVideos, maxVideoRefs, maxVideoRefDuration, setReferenceVideos],
);
const maxAudioRefs = selectedModel?.audio_references_max ?? 2;
const maxAudioRefDurationTotal =
selectedModel?.audio_references_max_total_duration_seconds ?? 30;
const audioRefPickerMax = Math.max(1, maxAudioRefs - referenceAudios.length);
const handleAudioRefPickerSelect = useCallback(
(id: string) => {
setAudioRefPickerSelectedIds((prev) => {
if (prev.includes(id)) return prev.filter((x) => x !== id);
if (prev.length >= audioRefPickerMax) {
return audioRefPickerMax === 1 ? [id] : prev;
}
return [...prev, id];
});
},
[audioRefPickerMax],
);
const handleLibraryAudioSelect = useCallback(
async (items: GalleryItem[]) => {
setIsAudioRefPickerOpen(false);
const availableSlots = Math.max(0, maxAudioRefs - referenceAudios.length);
const picked = items.slice(0, availableSlots);
const added: RefAudio[] = [];
let total = referenceAudios.reduce((sum, a) => sum + a.duration, 0);
for (const item of picked) {
const url = item.fullImage;
if (!url) continue;
const duration =
item.durationMillis != null
? Math.round(item.durationMillis / 1000)
: await getAudioDurationFromUrl(url);
if (duration <= 0) {
toast.error("Could not read audio file");
continue;
}
if (total + duration > maxAudioRefDurationTotal) {
toast.error(
`Audio too long — max ${maxAudioRefDurationTotal}s total (${maxAudioRefDurationTotal - total}s remaining)`,
);
continue;
}
total += duration;
added.push({
id: Math.random().toString(36).substring(7),
url,
file: new File([], "library-audio"),
mediaToken: item.id,
duration,
});
}
if (added.length > 0) {
setReferenceAudios([...referenceAudios, ...added]);
}
},
[
referenceAudios,
maxAudioRefs,
maxAudioRefDurationTotal,
setReferenceAudios,
],
);
const handleEndFrameLibrarySelect = useCallback((items: GalleryItem[]) => {
const item = items[0];
if (!item) return;
@@ -1161,6 +1234,7 @@ export default function CreateVideo() {
maxAudioRefDuration={
selectedModel?.audio_references_max_total_duration_seconds ?? 30
}
onPickAudioFromLibrary={() => setIsAudioRefPickerOpen(true)}
/>
) : undefined;
@@ -1487,7 +1561,9 @@ export default function CreateVideo() {
richList
triggerIcon={
<img
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
src={getCreatorIconPathForModelId(
selectedModel?.model ?? "",
)}
alt=""
className="h-4 w-4 icon-auto-contrast"
/>
@@ -1504,7 +1580,28 @@ export default function CreateVideo() {
})
}
mentionItems={mentionItems.length > 0 ? mentionItems : undefined}
mediaReferenceRow={mediaReferenceRow}
videoRefsSupported={supportsVideoRefs}
referenceVideos={referenceVideos}
onReferenceVideosChange={setReferenceVideos}
maxVideoCount={maxVideoRefs}
maxVideoRefDuration={maxVideoRefDuration}
onPickVideoFromLibrary={
supportsVideoRefs
? () => setIsVideoRefPickerOpen(true)
: undefined
}
audioRefsSupported={supportsAudioRefs}
referenceAudios={referenceAudios}
onReferenceAudiosChange={setReferenceAudios}
maxAudioCount={selectedModel?.audio_references_max ?? 2}
maxAudioRefDuration={
selectedModel?.audio_references_max_total_duration_seconds ?? 30
}
onPickAudioFromLibrary={
supportsAudioRefs
? () => setIsAudioRefPickerOpen(true)
: undefined
}
rightToolbar={
<GenerationCountPicker
batchSizeMax={selectedModel?.batch_size_max ?? 4}
@@ -1693,6 +1790,17 @@ export default function CreateVideo() {
forceFilter="video"
hideFilter
/>
<GalleryModal
mode="select"
isOpen={isAudioRefPickerOpen}
onClose={() => setIsAudioRefPickerOpen(false)}
selectedItemIds={audioRefPickerSelectedIds}
onSelectItem={handleAudioRefPickerSelect}
maxSelections={audioRefPickerMax}
onUseSelected={handleLibraryAudioSelect}
forceFilter="audio"
hideFilter
/>
<CharactersModal
isOpen={isCharactersModalOpen}
onClose={() => setIsCharactersModalOpen(false)}
@@ -2,3 +2,4 @@ export * from "./lib/button";
export * from "./lib/ToggleButton";
export * from "./lib/FilterButtons";
export * from "./lib/GenerateButton";
export * from "./lib/GenerateIconButton";
@@ -0,0 +1,60 @@
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faArrowUp,
faCoins,
faSpinnerThird,
} from "@fortawesome/pro-solid-svg-icons";
import { ButtonHTMLAttributes } from "react";
import { twMerge } from "tailwind-merge";
interface GenerateIconButtonProps
extends Omit<ButtonHTMLAttributes<HTMLButtonElement>, "children"> {
credits?: number | null;
loading?: boolean;
}
/**
* Compact generate control: credit cost as plain text next to a circular
* arrow-up button. Desktop promptboxes use this; the mobile form keeps the
* labeled `GenerateButton` pill.
*/
export const GenerateIconButton = ({
credits,
loading,
className,
disabled,
...rest
}: GenerateIconButtonProps) => {
const isDisabled = disabled || loading;
return (
<div className={twMerge("flex shrink-0 items-center gap-2.5", className)}>
{credits != null && (
<span
className={twMerge(
"flex items-center gap-1.5 text-[13px] font-semibold tabular-nums text-base-fg/80 transition-opacity",
isDisabled && "opacity-50",
)}
title={`${credits} credit${credits !== 1 ? "s" : ""} cost`}
>
<FontAwesomeIcon icon={faCoins} className="text-xs" />
{credits}
</span>
)}
<button
type="button"
className="flex h-9 w-9 items-center justify-center rounded-full bg-primary text-white shadow-sm transition-all duration-150 hover:bg-primary-400 active:scale-95 disabled:cursor-not-allowed disabled:opacity-40 disabled:active:scale-100"
disabled={isDisabled}
{...rest}
>
<FontAwesomeIcon
icon={loading ? faSpinnerThird : faArrowUp}
className={loading ? "animate-spin" : undefined}
/>
</button>
</div>
);
};
export default GenerateIconButton;
@@ -6,7 +6,12 @@ import React, {
} from "react";
import { createPortal } from "react-dom";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faCheck, faEllipsis, faUpload } from "@fortawesome/pro-solid-svg-icons";
import {
faCheck,
faEllipsis,
faMusic,
faUpload,
} from "@fortawesome/pro-solid-svg-icons";
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
import { twMerge } from "tailwind-merge";
import galleryDnd from "./galleryDnd";
@@ -43,6 +48,9 @@ interface GalleryDraggableItemProps {
selected: boolean;
onClick: () => void;
onImageError: (e: React.SyntheticEvent<HTMLImageElement>) => void;
/** Resolved prompt text for audio items — shown on the tile since audio has
* no thumbnail to speak for itself. */
audioPromptText?: string;
disableTooltipAndBadge?: boolean;
imageFit?: "cover" | "contain";
onDeleted?: (id: string) => void;
@@ -68,6 +76,7 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
selected,
onClick,
onImageError,
audioPromptText,
disableTooltipAndBadge = false,
imageFit = "cover",
onDeleted,
@@ -92,6 +101,9 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
// preview GIF, so the thumbnail URL 404s for a while. Show a spinner and
// refresh the image every 5s until it loads.
const isVideo = item.mediaClass === "video";
// Audio tiles are click-only: there is no canvas drop target for audio, so
// they never start a gallery drag.
const isAudio = item.mediaClass === "audio";
const [retryAttempt, setRetryAttempt] = useState(0);
const [videoLoaded, setVideoLoaded] = useState(false);
@@ -191,6 +203,7 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
// No drag tracking on touch: a finger drag should scroll the page, and a
// tap still clicks via the button's onClick.
if (event.pointerType === "touch") return;
if (isAudio) return;
dragStarted.current = false;
const moveListener = (moveEvent: PointerEvent) => {
const dx = moveEvent.pageX - event.pageX;
@@ -244,7 +257,7 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
: "border-transparent hover:border-primary",
mode === "select"
? "cursor-pointer"
: disableTooltipAndBadge && !bulkSelectionMode
: (disableTooltipAndBadge && !bulkSelectionMode) || isAudio
? "cursor-pointer"
: "cursor-grab hover:cursor-grab active:cursor-grabbing",
)}
@@ -253,7 +266,21 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
aria-label={item.label}
>
<div className="relative h-full w-full">
{!item.thumbnail ? (
{isAudio ? (
<div className="flex h-full w-full flex-col bg-gradient-to-br from-white/[0.08] via-white/[0.04] to-white/[0.02]">
<div className="flex min-h-0 flex-1 items-center justify-center">
<div className="flex h-11 w-11 items-center justify-center rounded-full bg-white/10 ring-1 ring-white/15">
<FontAwesomeIcon
icon={faMusic}
className="text-lg text-white/70"
/>
</div>
</div>
<p className="line-clamp-2 shrink-0 px-2.5 pb-2.5 text-left text-xs leading-snug text-white/75">
{audioPromptText || item.label}
</p>
</div>
) : !item.thumbnail ? (
<div className="flex h-full w-full items-center justify-center bg-black/30">
<span className="text-white/60">Image not available</span>
</div>
@@ -427,16 +454,18 @@ export const GalleryDraggableItem: React.FC<GalleryDraggableItemProps> = ({
className="-mt-3 bg-ui-controls text-base-fg border border-ui-panel-border"
content={
<div className="flex flex-col items-center text-xs whitespace-nowrap">
<span>
<span className="font-bold">Drag</span>
<span className="opacity-50">
{item.mediaClass === "dimensional" ||
item.mediaClass === "mesh" ||
item.mediaClass === "splat"
? " to add to scene"
: " to add"}
{!isAudio && (
<span>
<span className="font-bold">Drag</span>
<span className="opacity-50">
{item.mediaClass === "dimensional" ||
item.mediaClass === "mesh" ||
item.mediaClass === "splat"
? " to add to scene"
: " to add"}
</span>
</span>
</span>
)}
<span>
<span className="font-bold">Click</span>
<span className="opacity-50"> to view</span>
@@ -58,6 +58,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faBorderAll,
faImage,
faMusic,
faVideo,
faCube,
faGlobe,
@@ -74,6 +75,10 @@ import {
faPencil,
faStar,
} from "@fortawesome/pro-solid-svg-icons";
import {
useMediaPromptTokens,
usePrompts,
} from "@storyteller/ui-generation-list";
import { SliderV2 } from "@storyteller/ui-sliderv2";
import { Tooltip } from "@storyteller/ui-tooltip";
import {
@@ -240,6 +245,10 @@ export interface GalleryItem {
batchImageToken?: string;
mediaTokens?: string[];
imageUrls?: string[];
/** Prompt token, when the list endpoint returns one (used by audio tiles). */
promptToken?: string;
/** Duration for audio/video items, when known. */
durationMillis?: number;
}
interface GroupedItems {
@@ -307,6 +316,7 @@ const SIDEBAR_FILTERS = [
{ id: "all", label: "All Assets", icon: faBorderAll },
{ id: "image", label: "Image", icon: faImage },
{ id: "video", label: "Video", icon: faVideo },
{ id: "audio", label: "Audio", icon: faMusic },
{ id: "mesh", label: "Mesh", icon: faCube },
{ id: "splat", label: "Splat", icon: faGlobe },
{ id: "uploaded", label: "Uploaded", icon: faUpload },
@@ -320,6 +330,8 @@ const getFilterMediaClass = (filter: string) => {
return [FilterMediaClasses.IMAGE];
case "video":
return [FilterMediaClasses.VIDEO];
case "audio":
return [FilterMediaClasses.AUDIO];
case "uploaded":
return [
FilterMediaClasses.IMAGE,
@@ -347,6 +359,8 @@ const getLabel = (item: any) => {
return "Image Generation";
case "video":
return "Video Generation";
case "audio":
return "Audio Generation";
case "dimensional":
case "mesh":
return "3D Object Generation";
@@ -741,6 +755,52 @@ export const GalleryModal = React.memo(
[activeFolderItems, groupItemsByDate],
);
// Audio tiles have no thumbnail, so they show their prompt text instead.
// Resolve it via the shared batched caches (media token → prompt token →
// prompt text) for audio items only; image/video walls skip the lookups.
const audioMediaTokensNeedingPrompt = useMemo(() => {
const tokens: string[] = [];
const collect = (item: GalleryItem) => {
if (item.mediaClass === "audio" && !item.promptToken) {
tokens.push(item.id);
}
};
allItems.forEach(collect);
activeFolderItems.forEach(collect);
return tokens;
}, [allItems, activeFolderItems]);
const resolvedAudioPromptTokens = useMediaPromptTokens(
audioMediaTokensNeedingPrompt,
);
const audioPromptTokens = useMemo(() => {
const tokens: string[] = [];
const collect = (item: GalleryItem) => {
if (item.mediaClass !== "audio") return;
const token =
item.promptToken || resolvedAudioPromptTokens.get(item.id);
if (token) tokens.push(token);
};
allItems.forEach(collect);
activeFolderItems.forEach(collect);
return tokens;
}, [allItems, activeFolderItems, resolvedAudioPromptTokens]);
const audioPromptsMap = usePrompts(audioPromptTokens);
const audioPromptTextFor = useCallback(
(item: GalleryItem): string | undefined => {
if (item.mediaClass !== "audio") return undefined;
const token =
item.promptToken || resolvedAudioPromptTokens.get(item.id);
return (
(token
? audioPromptsMap.get(token)?.maybe_positive_prompt
: undefined) || undefined
);
},
[resolvedAudioPromptTokens, audioPromptsMap],
);
const handleImageError = useCallback((url: string) => {
console.error(`Failed to load gallery modal image: ${url}`);
failedImageUrls.current.add(url);
@@ -815,7 +875,9 @@ export const GalleryModal = React.memo(
id: item.token,
label: getLabel(item),
thumbnail:
item.media_class === "video"
item.media_class === "audio"
? null
: item.media_class === "video"
? item.media_links.maybe_video_previews?.animated
: is3DClass(item.media_class)
? // Prefer the modern CDN link; fall back to the deprecated
@@ -842,6 +904,8 @@ export const GalleryModal = React.memo(
: item.media_class === "splat"
? "splat"
: undefined,
promptToken: item.maybe_prompt_token ?? undefined,
durationMillis: item.maybe_duration_millis ?? undefined,
}),
[],
);
@@ -854,13 +918,15 @@ export const GalleryModal = React.memo(
id: item.token,
label: getLabel(item),
thumbnail:
is3DClass(item.media_class)
? (item.cover_image?.maybe_links?.cdn_url ??
item.cover_image?.maybe_cover_image_public_bucket_url ??
null)
: getMediaThumbnail(item.media_links, item.media_class, {
size: THUMBNAIL_SIZES.MEDIUM,
}),
item.media_class === "audio"
? null
: is3DClass(item.media_class)
? (item.cover_image?.maybe_links?.cdn_url ??
item.cover_image?.maybe_cover_image_public_bucket_url ??
null)
: getMediaThumbnail(item.media_links, item.media_class, {
size: THUMBNAIL_SIZES.MEDIUM,
}),
thumbnailUrlTemplate:
item.media_links?.maybe_thumbnail_template ?? undefined,
fullImage: item.media_links?.cdn_url ?? null,
@@ -868,6 +934,8 @@ export const GalleryModal = React.memo(
mediaClass: item.media_class || "image",
isUpload: !!item.is_user_upload,
batchImageToken: item.maybe_batch_token ?? undefined,
promptToken: item.maybe_prompt_token ?? undefined,
durationMillis: item.maybe_duration_millis ?? undefined,
}),
[],
);
@@ -949,7 +1017,8 @@ export const GalleryModal = React.memo(
currentFilter === "uploaded" ||
currentFilter === "all" ||
currentFilter === "image" ||
currentFilter === "video",
currentFilter === "video" ||
currentFilter === "audio",
user_uploads_only: currentFilter === "uploaded",
page_index: reset ? 0 : pageIndexRef.current,
page_size: PAGE_SIZE,
@@ -2088,11 +2157,9 @@ export const GalleryModal = React.memo(
if (activeFilter === "splat") return item.mediaClass === "splat";
if (activeFilter === "image") return item.mediaClass === "image";
if (activeFilter === "video") return item.mediaClass === "video";
if (activeFilter === "audio") return item.mediaClass === "audio";
if (activeFilter === "all") {
return (
item.mediaClass !== "audio" &&
(item as any).mediaType !== "scene_json"
);
return (item as any).mediaType !== "scene_json";
}
return true;
},
@@ -2422,6 +2489,8 @@ export const GalleryModal = React.memo(
? maxSelections === 1
? "Select Video"
: "Select Videos"
: activeFilter === "audio"
? "Select Audio"
: activeFilter === "mesh"
? maxSelections === 1
? "Select 3D Object"
@@ -2765,6 +2834,7 @@ export const GalleryModal = React.memo(
item={item}
mode={mode}
activeFilter={activeFilter}
audioPromptText={audioPromptTextFor(item)}
selected={selectedItemIds.includes(item.id)}
onClick={() => handleItemClick(item)}
onImageError={(e) => {
@@ -2909,6 +2979,9 @@ export const GalleryModal = React.memo(
item={item}
mode={mode}
activeFilter={activeFilter}
audioPromptText={audioPromptTextFor(
item,
)}
selected={selectedItemIds.includes(
item.id,
)}
@@ -3009,11 +3082,13 @@ export const GalleryModal = React.memo(
const placeholderIcon =
si.mediaClass === "video"
? faVideo
: si.mediaClass === "dimensional" ||
si.mediaClass === "mesh" ||
si.mediaClass === "splat"
? faCube
: faImage;
: si.mediaClass === "audio"
? faMusic
: si.mediaClass === "dimensional" ||
si.mediaClass === "mesh" ||
si.mediaClass === "splat"
? faCube
: faImage;
return (
<BulkThumb
key={si.id}
@@ -9,22 +9,19 @@
"path": "../slider-v2"
},
{
"path": "../../common"
},
{
"path": "../../api"
"path": "../generation-list"
},
{
"path": "../close-button"
},
{
"path": "../button"
},
{
"path": "../lightbox-modal"
},
{
"path": "../modal"
"path": "../../common"
},
{
"path": "../../api"
},
{
"path": "../toaster"
@@ -41,6 +38,12 @@
{
"path": "../loading-spinner"
},
{
"path": "../button"
},
{
"path": "../modal"
},
{
"path": "./tsconfig.lib.json"
},
@@ -50,22 +50,19 @@
"path": "../slider-v2/tsconfig.lib.json"
},
{
"path": "../../common/tsconfig.lib.json"
},
{
"path": "../../api/tsconfig.lib.json"
"path": "../generation-list/tsconfig.lib.json"
},
{
"path": "../close-button/tsconfig.lib.json"
},
{
"path": "../button/tsconfig.lib.json"
},
{
"path": "../lightbox-modal/tsconfig.lib.json"
},
{
"path": "../modal/tsconfig.lib.json"
"path": "../../common/tsconfig.lib.json"
},
{
"path": "../../api/tsconfig.lib.json"
},
{
"path": "../toaster/tsconfig.lib.json"
@@ -81,6 +78,12 @@
},
{
"path": "../loading-spinner/tsconfig.lib.json"
},
{
"path": "../button/tsconfig.lib.json"
},
{
"path": "../modal/tsconfig.lib.json"
}
]
}
@@ -161,30 +161,25 @@ export const GalleryCard = memo(function GalleryCard({
style={{ contentVisibility: "auto", containIntrinsicSize: "auto 200px" }}
>
{isAudio ? (
<div className="flex h-full flex-col bg-gradient-to-br from-white/[0.07] to-white/[0.02] p-3">
<div className="flex items-start gap-2">
<p className="line-clamp-3 min-w-0 flex-1 text-sm leading-snug text-white/85">
{title || item.label}
</p>
<FontAwesomeIcon
icon={faMusic}
className="pointer-events-none mt-0.5 shrink-0 text-base text-white/15"
/>
<div className="flex h-full flex-col bg-gradient-to-br from-white/[0.08] via-white/[0.04] to-white/[0.02] p-3">
<p className="line-clamp-2 shrink-0 text-sm leading-snug text-white/85">
{title || item.label}
</p>
<div className="flex min-h-0 flex-1 items-center justify-center">
<div className="flex h-12 w-12 items-center justify-center rounded-full bg-white/10 ring-1 ring-white/15">
<FontAwesomeIcon
icon={faMusic}
className="text-xl text-white/70"
/>
</div>
</div>
{item.fullImage ? (
<div className="flex min-h-0 flex-1 items-center">
<div className="w-full">
<WaveformAudioPlayer
src={item.fullImage}
durationMillis={item.durationMillis}
/>
</div>
{item.fullImage && (
<div className="w-full shrink-0">
<WaveformAudioPlayer
src={item.fullImage}
durationMillis={item.durationMillis}
/>
</div>
) : (
<FontAwesomeIcon
icon={faMusic}
className="m-auto text-2xl text-white/20"
/>
)}
</div>
) : (
@@ -614,11 +614,13 @@ export function LightboxModal({
</video>
) : mediaClass === "audio" ? (
<div className="flex h-full w-full items-center justify-center px-4 sm:px-8">
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-white/[0.04] px-2 py-8">
<FontAwesomeIcon
icon={faMusic}
className="mx-auto mb-5 block text-3xl text-white/20"
/>
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-gradient-to-br from-white/[0.08] via-white/[0.04] to-white/[0.02] px-4 py-10 sm:px-6">
<div className="mx-auto mb-6 flex h-16 w-16 items-center justify-center rounded-full bg-white/10 ring-1 ring-white/15">
<FontAwesomeIcon
icon={faMusic}
className="text-2xl text-white/70"
/>
</div>
<WaveformAudioPlayer
key={selectedImageUrl as string}
src={selectedImageUrl as string}
@@ -13,3 +13,8 @@ export * from "./lib/PromptBoxErrorBoundary";
export * from "./lib/common/AudioTuningPopover";
export * from "./lib/common/SoundsSettingsPopover";
export * from "./lib/common/StylePromptRow";
export * from "./lib/deck/deckTypes";
export * from "./lib/deck/DeckCard";
export * from "./lib/deck/ReferenceDeck";
export * from "./lib/deck/KeyframeCards";
export * from "./lib/deck/useDeckMedia";
@@ -1,5 +1,9 @@
import { useMemo, useRef, useState, ReactNode } from "react";
import { toast } from "@storyteller/ui-toaster";
import {
GalleryModal,
type GalleryItem,
} from "@storyteller/ui-gallery-modal";
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { GenerateButton, ToggleButton } from "@storyteller/ui-button";
@@ -23,7 +27,11 @@ import {
getModelDescription,
getModelInfo,
} from "@storyteller/model-list";
import { usePromptAudioStore, useEnterToGenerateStore } from "./promptStore";
import {
usePromptAudioStore,
useEnterToGenerateStore,
type RefAudio,
} from "./promptStore";
import { useAutoGrowEditorHeight } from "./useAutoGrowEditorHeight";
import {
PromptFullscreenModal,
@@ -44,6 +52,16 @@ const AUDIO_REF_MAX_DURATION_SECONDS = 600;
// Capability flags are serde-skipped when absent — only `true` counts.
const supports = (flag: boolean | null | undefined): boolean => flag === true;
// Resolves 0 when metadata can't be loaded.
const getAudioDurationFromSrc = (src: string): Promise<number> =>
new Promise((resolve) => {
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.onloadedmetadata = () => resolve(Math.round(audio.duration));
audio.onerror = () => resolve(0);
audio.src = src;
});
interface PromptBoxAudioProps {
// Audio models from GET /v1/omni_gen/models/audio (useOmniGenAudioModels).
models: OmniGenAudioModelDetails[];
@@ -95,6 +113,14 @@ export const PromptBoxAudio = ({
const [isFocused, setIsFocused] = useState(false);
const enterToGenerate = useEnterToGenerateStore((s) => s.enabled);
// Audio library picker (the "From library" button in the reference row).
const [isAudioLibraryOpen, setIsAudioLibraryOpen] = useState(false);
const [audioLibrarySelectedIds, setAudioLibrarySelectedIds] = useState<
string[]
>([]);
const [isAudioLibraryProcessing, setIsAudioLibraryProcessing] =
useState(false);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const { isExpanded, toggleExpand } = useAutoGrowEditorHeight(
textareaRef,
@@ -178,6 +204,75 @@ export const PromptBoxAudio = ({
setReferenceAudios(audios);
};
const maxAudioLibrarySelections = Math.max(
1,
maxAudioRefs - referenceAudios.length,
);
const handleAudioLibrarySelectToggle = (id: string) => {
setAudioLibrarySelectedIds((prev) => {
if (prev.includes(id)) return prev.filter((x) => x !== id);
if (prev.length >= maxAudioLibrarySelections) {
return maxAudioLibrarySelections === 1 ? [id] : prev;
}
return [...prev, id];
});
};
const closeAudioLibrary = () => {
setIsAudioLibraryOpen(false);
setAudioLibrarySelectedIds([]);
};
const handleAudioLibraryUseSelected = async (items: GalleryItem[]) => {
const availableSlots = Math.max(0, maxAudioRefs - referenceAudios.length);
const picked = items
.slice(0, availableSlots)
.filter((item): item is GalleryItem & { fullImage: string } =>
Boolean(item.fullImage),
);
setIsAudioLibraryProcessing(true);
try {
// Use the duration the list endpoint already knows; probe the file's
// metadata only when it doesn't.
const durations = await Promise.all(
picked.map((item) =>
item.durationMillis != null
? Promise.resolve(Math.round(item.durationMillis / 1000))
: getAudioDurationFromSrc(item.fullImage),
),
);
const added: RefAudio[] = [];
let total = referenceAudios.reduce((sum, a) => sum + a.duration, 0);
for (let i = 0; i < picked.length; i++) {
const item = picked[i]!;
const duration = durations[i]!;
if (total + duration > AUDIO_REF_MAX_DURATION_SECONDS) {
toast.error(
`Total audio duration cannot exceed ${AUDIO_REF_MAX_DURATION_SECONDS}s`,
);
break;
}
total += duration;
added.push({
id: Math.random().toString(36).substring(7),
url: item.fullImage,
file: new File([], "library-audio"),
mediaToken: item.id,
duration,
});
}
if (added.length > 0) {
handleReferenceAudiosChange([...referenceAudios, ...added]);
}
} finally {
setIsAudioLibraryProcessing(false);
}
closeAudioLibrary();
};
const handleReferenceImagesChange = (images: typeof referenceImages) => {
if (images.length > 0 && referenceAudios.length > 0) {
setReferenceAudios([]);
@@ -383,6 +478,9 @@ export const PromptBoxAudio = ({
maxAudioCount={audioRefsSupported ? maxAudioRefs : 0}
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
uploadAudio={uploadAudio}
onPickAudioFromLibrary={
audioRefsSupported ? () => setIsAudioLibraryOpen(true) : undefined
}
audioRequired={requiresAudioRef}
imageSupported={imageRefsSupported}
referenceImages={referenceImages}
@@ -471,6 +569,18 @@ export const PromptBoxAudio = ({
</div>
</div>
</div>
<GalleryModal
mode="select"
isOpen={isAudioLibraryOpen}
onClose={closeAudioLibrary}
selectedItemIds={audioLibrarySelectedIds}
onSelectItem={handleAudioLibrarySelectToggle}
maxSelections={maxAudioLibrarySelections}
onUseSelected={handleAudioLibraryUseSelected}
useSelectedLoading={isAudioLibraryProcessing}
forceFilter="audio"
hideFilter
/>
<PromptFullscreenModal
isOpen={isFullscreen}
onClose={closeFullscreen}
@@ -1,11 +1,10 @@
import { useState, useRef, useEffect, ReactNode } from "react";
import { useState, useRef, useEffect, useMemo, ReactNode } from "react";
import { useSignals } from "@preact/signals-react/runtime";
import { JobContextType, UploaderState } from "@storyteller/common";
import { toast } from "@storyteller/ui-toaster";
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { Button, GenerateButton } from "@storyteller/ui-button";
import { Modal } from "@storyteller/ui-modal";
import { GenerateIconButton } from "@storyteller/ui-button";
import { GenerateImage, GenerateImageRequest } from "@storyteller/tauri-api";
import {
faExpand,
@@ -14,6 +13,8 @@ import {
} from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { ImageModel } from "@storyteller/model-list";
import { arrayMove } from "@dnd-kit/sortable";
import type { UploadMediaFn } from "@storyteller/api";
import {
usePromptImageStore,
RefImage,
@@ -24,13 +25,15 @@ import { PromptFullscreenModal, useFullscreenPrompt } from "./PromptFullscreenMo
import { PromptFullscreenButton } from "./PromptFullscreenButton";
import { gtagEvent } from "@storyteller/google-analytics";
import { twMerge } from "tailwind-merge";
import { ImagePromptRow } from "./ImagePromptRow";
import { GenerationProvider } from "@storyteller/api-enums";
import { AspectRatioPicker } from "./common/AspectRatioPicker";
import { AspectRatioIcon } from "./common/AspectRatioIcon";
import { GenerationCountPicker } from "./common/GenerationCountPicker";
import { ResolutionPicker } from "./common/ResolutionPicker";
import { QualityPicker } from "./common/QualityPicker";
import { ReferenceDeck } from "./deck/ReferenceDeck";
import { useDeckMedia } from "./deck/useDeckMedia";
import { DeckAddAction, DeckItem } from "./deck/deckTypes";
interface PromptBoxImageProps {
useJobContext: () => JobContextType;
@@ -67,7 +70,6 @@ export const PromptBoxImage = ({
selectedProvider,
imageMediaId,
url,
onImageRowVisibilityChange,
credits,
modelSelector,
}: PromptBoxImageProps) => {
@@ -91,9 +93,6 @@ export const PromptBoxImage = ({
}
}, [imageMediaId, url]);
const [isModalOpen, setIsModalOpen] = useState(false);
const [content, setContent] = useState<ReactNode>(null);
const prompt = usePromptImageStore((s) => s.prompt);
const setPrompt = usePromptImageStore((s) => s.setPrompt);
const aspectRatio = usePromptImageStore((s) => s.aspectRatio);
@@ -118,14 +117,74 @@ export const PromptBoxImage = ({
const referenceImages = usePromptImageStore((s) => s.referenceImages);
const setReferenceImages = usePromptImageStore((s) => s.setReferenceImages);
const enterToGenerate = useEnterToGenerateStore((s) => s.enabled);
const [uploadingImages, _setUploadingImages] = useState<
{ id: string; file: File }[]
>([]);
const [showImagePrompts, setShowImagePrompts] = useState(false);
const isImageRowVisible =
showImagePrompts ||
referenceImages.length > 0 ||
uploadingImages.length > 0;
const maxImagePromptCount = Math.max(
1,
selectedModel?.maxImagePromptCount ?? 1,
);
const deck = useDeckMedia({
referenceImages,
setReferenceImages,
maxImages: maxImagePromptCount,
uploadImage: uploadImage as UploadMediaFn | undefined,
ownGalleryModal: true,
});
const deckItems: DeckItem[] = useMemo(
() => [
...referenceImages.map((img, i) => ({
id: img.id,
kind: "image" as const,
url: img.url,
name: `Image ${i + 1}`,
})),
...deck.uploadingImages.map((entry, i) => ({
id: entry.id,
kind: "image" as const,
url: entry.previewUrl,
name: `Image ${referenceImages.length + i + 1}`,
uploading: true,
})),
],
[referenceImages, deck.uploadingImages],
);
const deckAddActions: DeckAddAction[] = [
{
key: "upload-image",
label: "Upload",
group: "image",
onSelect: deck.openImageUpload,
},
{
key: "library-image",
label: "Pick from library",
group: "image",
onSelect: () => deck.openGallery("start"),
},
];
const renderReferenceDeck = (alwaysExpanded?: boolean) =>
selectedModel?.canUseImagePrompt ? (
<ReferenceDeck
items={deckItems}
canAdd={deckItems.length < maxImagePromptCount}
addActions={deckAddActions}
addMenuGroupHints={{
image: `${referenceImages.length}/${maxImagePromptCount}`,
}}
onAddClick={deck.openAnyUpload}
onRemove={(id) =>
setReferenceImages(referenceImages.filter((img) => img.id !== id))
}
onReorderImages={(from, to) =>
setReferenceImages(arrayMove(referenceImages, from, to))
}
onClearAll={() => setReferenceImages([])}
alwaysExpanded={alwaysExpanded}
/>
) : null;
// New aspect ratio and resolution — stored globally so cost estimates can observe them
const commonAspectRatio = usePromptImageStore((s) => s.commonAspectRatio);
@@ -137,9 +196,6 @@ export const PromptBoxImage = ({
const commonQuality = usePromptImageStore((s) => s.commonQuality);
const setCommonQuality = usePromptImageStore((s) => s.setCommonQuality);
useEffect(() => {
onImageRowVisibilityChange?.(isImageRowVisible);
}, [isImageRowVisible, onImageRowVisibilityChange]);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "Wide",
@@ -362,88 +418,20 @@ export const PromptBoxImage = ({
return (
<>
<Modal
isOpen={isModalOpen}
onClose={() => {
setIsModalOpen(false);
setContent(null);
}}
className="max-w-4xl max-h-[80vh]"
>
{content}
</Modal>
{deck.fileInputs}
{deck.galleryModal}
<div className="relative z-20 flex flex-col">
{isImageRowVisible && selectedModel?.canUseImagePrompt && (
<ImagePromptRow
visible={true}
maxImagePromptCount={Math.max(
1,
selectedModel?.maxImagePromptCount ?? 1,
)}
allowUpload={true}
referenceImages={referenceImages}
setReferenceImages={setReferenceImages}
onVisibilityChange={onImageRowVisibilityChange}
className=""
uploadImage={uploadImage as any}
onImageClick={(image) => {
setContent(
<img
src={image.url}
alt="Reference preview"
className="w-full h-full object-contain"
/>,
);
setIsModalOpen(true);
}}
/>
)}
<div
className={twMerge(
"glass relative w-full rounded-2xl p-4",
isImageRowVisible &&
selectedModel?.canUseImagePrompt &&
"rounded-t-none",
isFocused
? "ring-1 ring-primary border-primary"
: "ring-1 ring-transparent",
)}
>
<div className="flex justify-center gap-2">
{selectedModel?.canUseImagePrompt && (
<Tooltip
content="Add Image"
position="top"
closeOnClick={true}
className={twMerge(isImageRowVisible && "hidden opacity-0")}
>
<Button
variant="action"
className={twMerge(
"h-8 w-8 p-0 bg-transparent hover:bg-transparent group transition-all border-0 shadow-none",
isImageRowVisible && "text-primary",
)}
onClick={() => setShowImagePrompts((prev) => !prev)}
>
<svg
width="24"
height="20"
viewBox="0 0 24 20"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className="group-hover:opacity-100 opacity-80 transition-all"
>
<path
opacity="1"
d="M2.66667 2H16C16.3667 2 16.6667 2.3 16.6667 2.66667V6.1125C17.1 6.04167 17.5458 6 18 6C18.225 6 18.4458 6.00833 18.6667 6.02917V2.66667C18.6667 1.19583 17.4708 0 16 0H2.66667C1.19583 0 0 1.19583 0 2.66667V16C0 17.4708 1.19583 18.6667 2.66667 18.6667H11.5C11.0625 18.0583 10.7083 17.3875 10.4542 16.6667H2.66667C2.3 16.6667 2 16.3667 2 16V2.66667C2 2.3 2.3 2 2.66667 2ZM11.8625 7.49167C11.6833 7.1875 11.3542 7 11 7C10.6458 7 10.3167 7.1875 10.1375 7.49167L8.2 10.7833L7.48333 9.75833C7.29583 9.49167 6.99167 9.33333 6.6625 9.33333C6.33333 9.33333 6.02917 9.49167 5.84167 9.75833L3.50833 13.0917C3.29583 13.3958 3.26667 13.7958 3.44167 14.125C3.61667 14.4542 3.9625 14.6667 4.33333 14.6667H10.0292C10.0125 14.4458 10 14.225 10 14C10 11.7833 10.9 9.77917 12.3542 8.33333L11.8625 7.49583V7.49167ZM5.33333 6.66667C6.07083 6.66667 6.66667 6.07083 6.66667 5.33333C6.66667 4.59583 6.07083 4 5.33333 4C4.59583 4 4 4.59583 4 5.33333C4 6.07083 4.59583 6.66667 5.33333 6.66667ZM18 20C21.3125 20 24 17.3125 24 14C24 10.6875 21.3125 8 18 8C14.6875 8 12 10.6875 12 14C12 17.3125 14.6875 20 18 20ZM18.6667 11.3333V13.3333H20.6667C21.0333 13.3333 21.3333 13.6333 21.3333 14C21.3333 14.3667 21.0333 14.6667 20.6667 14.6667H18.6667V16.6667C18.6667 17.0333 18.3667 17.3333 18 17.3333C17.6333 17.3333 17.3333 17.0333 17.3333 16.6667V14.6667H15.3333C14.9667 14.6667 14.6667 14.3667 14.6667 14C14.6667 13.6333 14.9667 13.3333 15.3333 13.3333H17.3333V11.3333C17.3333 10.9667 17.6333 10.6667 18 10.6667C18.3667 10.6667 18.6667 10.9667 18.6667 11.3333Z"
fill="currentColor"
/>
</svg>
</Button>
</Tooltip>
)}
{renderReferenceDeck()}
<div className="promptbox-resize-wrap relative flex-1">
<textarea
@@ -537,16 +525,12 @@ export const PromptBoxImage = ({
setGenerationCount(count);
}}
/>
<GenerateButton
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={undefined}
<GenerateIconButton
onClick={handleEnqueue}
disabled={!prompt.trim()}
loading={isEnqueueing}
credits={credits}
>
Generate
</GenerateButton>
/>
</div>
</div>
<div className="absolute -bottom-1 left-1/2 -translate-x-1/2">
@@ -575,24 +559,7 @@ export const PromptBoxImage = ({
promptLength={prompt.length}
maxLength={maxLen}
footerControls={modelSelector}
imagePromptRow={
selectedModel?.canUseImagePrompt ? (
<ImagePromptRow
visible={true}
maxImagePromptCount={Math.max(
1,
selectedModel?.maxImagePromptCount ?? 1,
)}
allowUpload={true}
referenceImages={referenceImages}
setReferenceImages={setReferenceImages}
uploadImage={uploadImage as any}
// Reset the inline row's absolute "float above the box"
// positioning so it sits in-flow in the modal, with rounded corners.
className="relative top-auto rounded-2xl"
/>
) : undefined
}
imagePromptRow={renderReferenceDeck(true) ?? undefined}
>
<textarea
placeholder="Describe what you want in the image..."
@@ -1,12 +1,10 @@
import { useState, useRef, useEffect, useMemo } from "react";
import { useSignals } from "@preact/signals-react/runtime";
import { JobContextType } from "@storyteller/common";
import { downloadFileFromUrl } from "@storyteller/api";
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { SliderV2 } from "@storyteller/ui-sliderv2";
import { Tooltip } from "@storyteller/ui-tooltip";
import { ToggleButton, GenerateButton } from "@storyteller/ui-button";
import { Modal } from "@storyteller/ui-modal";
import { ToggleButton, GenerateIconButton } from "@storyteller/ui-button";
import { GenerateVideo, GenerateVideoRequest } from "@storyteller/tauri-api";
import {
faWaveformLines,
@@ -16,7 +14,7 @@ import {
} from "@fortawesome/pro-solid-svg-icons";
import { faCircleInfo } from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import { arrayMove } from "@dnd-kit/sortable";
import {
CommonResolution,
effectivePromptMaxLength,
@@ -33,8 +31,11 @@ import {
useEnterToGenerateStore,
} from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
import { ImagePromptRow } from "./ImagePromptRow";
import type { UploadImageFn } from "./ImagePromptRow";
import { ReferenceDeck } from "./deck/ReferenceDeck";
import { KeyframeCards } from "./deck/KeyframeCards";
import { useDeckMedia } from "./deck/useDeckMedia";
import { DeckAddAction, DeckItem } from "./deck/deckTypes";
import { AspectRatioIcon } from "./common/AspectRatioIcon";
import { VideoGenerationCountPicker } from "./common/VideoGenerationCountPicker";
import { twMerge } from "tailwind-merge";
@@ -127,7 +128,6 @@ export const PromptBoxVideo = ({
selectedProvider,
imageMediaId,
url,
onImageRowVisibilityChange,
uploadImage,
uploadVideo,
uploadAudio,
@@ -149,8 +149,6 @@ export const PromptBoxVideo = ({
}
}, [imageMediaId, url]);
const [isModalOpen, setIsModalOpen] = useState(false);
const [content, setContent] = useState<React.ReactNode>(null);
const prompt = usePromptVideoStore((s) => s.prompt);
const setPrompt = usePromptVideoStore((s) => s.setPrompt);
const generateWithSound = usePromptVideoStore((s) => s.generateWithSound);
@@ -239,9 +237,6 @@ export const PromptBoxVideo = ({
});
};
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[],
);
const referenceImages = usePromptVideoStore((s) => s.referenceImages);
const setReferenceImages = usePromptVideoStore((s) => s.setReferenceImages);
const endFrameImage = usePromptVideoStore((s) => s.endFrameImage);
@@ -250,14 +245,6 @@ export const PromptBoxVideo = ({
const setReferenceVideos = usePromptVideoStore((s) => s.setReferenceVideos);
const referenceAudios = usePromptVideoStore((s) => s.referenceAudios);
const setReferenceAudios = usePromptVideoStore((s) => s.setReferenceAudios);
const [uploadingImages, _setUploadingImages] = useState<
{ id: string; file: File }[]
>([]);
const [showImagePrompts, _setShowImagePrompts] = useState(true);
const isImageRowVisible =
showImagePrompts ||
referenceImages.length > 0 ||
uploadingImages.length > 0;
// TODO: Get rid of default resolutions. Just disable it if not present.
let aspectRatioOptions: PopoverItem[];
@@ -282,8 +269,7 @@ export const PromptBoxVideo = ({
aspectRatioOptions = buildAspectRatioOptions(DEFAULT_RESOLUTIONS);
}
const [, setAspectRatioList] =
useState<PopoverItem[]>(aspectRatioOptions);
const [, setAspectRatioList] = useState<PopoverItem[]>(aspectRatioOptions);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const mentionEditorRef = useRef<HTMLDivElement>(null);
@@ -356,10 +342,6 @@ export const PromptBoxVideo = ({
}
}, [imageMediaId, url]);
useEffect(() => {
onImageRowVisibilityChange?.(isImageRowVisible);
}, [isImageRowVisible, onImageRowVisibilityChange]);
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatio(selectedItem.label);
setAspectRatioList((prev) =>
@@ -467,7 +449,7 @@ export const PromptBoxVideo = ({
selected: inputMode === "keyframe",
},
{
label: "Reference",
label: "Omni Reference",
description: "Multi-media ref",
selected: inputMode === "reference",
},
@@ -476,7 +458,7 @@ export const PromptBoxVideo = ({
const handleInputModeSelect = (selectedItem: PopoverItem) => {
const mode: VideoInputMode =
selectedItem.label === "Reference" ? "reference" : "keyframe";
selectedItem.label === "Omni Reference" ? "reference" : "keyframe";
setInputMode(mode);
// Clear images/videos when switching modes to avoid stale state
if (mode === "reference") {
@@ -493,6 +475,265 @@ export const PromptBoxVideo = ({
? (selectedModel?.maxReferenceImages ?? 3)
: 1;
const maxVideoCount = selectedModel?.maxReferenceVideos ?? 3;
const maxAudioCount = selectedModel?.maxReferenceAudios ?? 2;
const deck = useDeckMedia({
referenceImages,
setReferenceImages,
maxImages: maxImageCount,
setEndFrameImage,
referenceVideos,
setReferenceVideos,
maxVideos: maxVideoCount,
maxVideoTotalSec: selectedModel?.maxVideoRefDuration ?? 15,
referenceAudios,
setReferenceAudios,
maxAudios: maxAudioCount,
maxAudioTotalSec: selectedModel?.maxAudioRefDuration ?? 15,
uploadImage,
uploadVideo,
uploadAudio,
ownGalleryModal: true,
});
// Mixed deck items ordered images → videos → audios: the @ImageN/@VideoN/
// @AudioN mention labels are index-derived per type, so this ordering (and
// image-only reordering) is load-bearing.
const deckItems: DeckItem[] = useMemo(
() => [
...referenceImages.map((img, i) => ({
id: img.id,
kind: "image" as const,
url: img.url,
name: `Image ${i + 1}`,
})),
...deck.uploadingImages.map((entry, i) => ({
id: entry.id,
kind: "image" as const,
url: entry.previewUrl,
name: `Image ${referenceImages.length + i + 1}`,
uploading: true,
})),
...referenceVideos.map((video, i) => ({
id: video.id,
kind: "video" as const,
url: video.url,
name: `Video ${i + 1}`,
duration: video.duration,
})),
...(deck.uploadingVideo
? [
{
id: deck.uploadingVideo.id,
kind: "video" as const,
url: deck.uploadingVideo.previewUrl,
name: `Video ${referenceVideos.length + 1}`,
uploading: true,
},
]
: []),
...referenceAudios.map((audio, i) => ({
id: audio.id,
kind: "audio" as const,
url: audio.url,
name: `Audio ${i + 1}`,
duration: audio.duration,
})),
...(deck.uploadingAudio
? [
{
id: deck.uploadingAudio.id,
kind: "audio" as const,
name: `Audio ${referenceAudios.length + 1}`,
uploading: true,
},
]
: []),
],
[
referenceImages,
referenceVideos,
referenceAudios,
deck.uploadingImages,
deck.uploadingVideo,
deck.uploadingAudio,
],
);
const refDeckAddActions: DeckAddAction[] = [];
if (referenceImages.length + deck.uploadingImages.length < maxImageCount) {
refDeckAddActions.push(
{
key: "upload-image",
label: "Upload",
group: "image",
onSelect: deck.openImageUpload,
},
{
key: "library-image",
label: "From library",
group: "image",
onSelect: () => deck.openGallery("start"),
},
);
}
if (referenceVideos.length < maxVideoCount && !deck.uploadingVideo) {
refDeckAddActions.push(
{
key: "upload-video",
label: "Upload",
group: "video",
onSelect: deck.openVideoUpload,
},
{
key: "library-video",
label: "From library",
group: "video",
onSelect: () => deck.openGallery("video"),
},
);
}
if (referenceAudios.length < maxAudioCount && !deck.uploadingAudio) {
refDeckAddActions.push(
{
key: "upload-audio",
label: "Upload",
group: "audio",
onSelect: deck.openAudioUpload,
},
{
key: "library-audio",
label: "From library",
group: "audio",
onSelect: () => deck.openGallery("audio"),
},
);
}
const handleRemoveDeckItem = (id: string) => {
if (referenceImages.some((img) => img.id === id)) {
setReferenceImages(referenceImages.filter((img) => img.id !== id));
} else if (referenceVideos.some((video) => video.id === id)) {
setReferenceVideos(referenceVideos.filter((video) => video.id !== id));
} else if (referenceAudios.some((audio) => audio.id === id)) {
setReferenceAudios(referenceAudios.filter((audio) => audio.id !== id));
}
};
const maxVideoTotalSec = selectedModel?.maxVideoRefDuration ?? 15;
const maxAudioTotalSec = selectedModel?.maxAudioRefDuration ?? 15;
const totalVideoRefSeconds = referenceVideos.reduce(
(sum, video) => sum + video.duration,
0,
);
const totalAudioRefSeconds = referenceAudios.reduce(
(sum, audio) => sum + audio.duration,
0,
);
const refDeckGroupHints = {
image: `${referenceImages.length}/${maxImageCount}`,
video: `${referenceVideos.length}/${maxVideoCount} · ${totalVideoRefSeconds}/${maxVideoTotalSec}s`,
audio: `${referenceAudios.length}/${maxAudioCount} · ${totalAudioRefSeconds}/${maxAudioTotalSec}s`,
};
const renderReferenceDeck = (alwaysExpanded?: boolean) => (
<ReferenceDeck
items={deckItems}
canAdd={refDeckAddActions.length > 0}
addActions={refDeckAddActions}
addMenuGroupHints={refDeckGroupHints}
onAddClick={deck.openAnyUpload}
onRemove={handleRemoveDeckItem}
onReorderImages={(from, to) =>
setReferenceImages(arrayMove(referenceImages, from, to))
}
onClearAll={() => {
setReferenceImages([]);
setReferenceVideos([]);
setReferenceAudios([]);
}}
alwaysExpanded={alwaysExpanded}
/>
);
const firstFrameItem: DeckItem | undefined = referenceImages[0]
? {
id: referenceImages[0].id,
kind: "image",
url: referenceImages[0].url,
name: "First frame",
}
: deck.uploadingImages[0]
? {
id: deck.uploadingImages[0].id,
kind: "image",
url: deck.uploadingImages[0].previewUrl,
name: "First frame",
uploading: true,
}
: undefined;
const lastFrameItem: DeckItem | undefined = endFrameImage
? {
id: endFrameImage.id,
kind: "image",
url: endFrameImage.url,
name: "Last frame",
}
: deck.uploadingEnd
? {
id: deck.uploadingEnd.id,
kind: "image",
url: deck.uploadingEnd.previewUrl,
name: "Last frame",
uploading: true,
}
: undefined;
const handleSwapFrames = () => {
const first = referenceImages[0];
if (!first || !endFrameImage) return;
setReferenceImages([endFrameImage]);
setEndFrameImage(first);
};
const renderKeyframeCards = () => (
<KeyframeCards
firstFrame={firstFrameItem}
lastFrame={lastFrameItem}
showLastFrame={!!selectedModel?.endFrame}
onFirstAddActions={[
{
key: "upload-first",
label: "Upload",
onSelect: deck.openImageUpload,
},
{
key: "library-first",
label: "Pick from library",
onSelect: () => deck.openGallery("start"),
},
]}
onLastAddActions={[
{
key: "upload-last",
label: "Upload",
onSelect: deck.openEndUpload,
},
{
key: "library-last",
label: "Pick from library",
onSelect: () => deck.openGallery("end"),
},
]}
onRemoveFirst={() => setReferenceImages([])}
onRemoveLast={() => setEndFrameImage(undefined)}
onSwap={handleSwapFrames}
/>
);
// Color palettes for @-mention highlights
const IMAGE_COLORS = [
"rgb(96, 165, 250)", // blue
@@ -962,8 +1203,6 @@ export const PromptBoxVideo = ({
}
};
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
const modelNeedsAnImageButNoneAreSelected =
selectedModel?.requiresImage && referenceImages.length === 0;
@@ -974,48 +1213,6 @@ export const PromptBoxVideo = ({
}
}, [selectedModel, endFrameImage, setEndFrameImage]);
// Shared reference-image row. Inline it floats above the box (default absolute
// positioning); the fullscreen modal passes a className to reset that so it
// sits in-flow above the controls with rounded corners.
const renderImagePromptRow = (className?: string) => (
<ImagePromptRow
visible={true}
isVideo={true}
isReferenceMode={isReferenceMode}
maxImagePromptCount={maxImageCount}
allowUpload={true}
referenceImages={referenceImages}
setReferenceImages={setReferenceImages}
onImageClick={(image) => {
setContent(
<img
src={image.url}
alt="Reference preview"
className="h-full w-full object-contain"
/>,
);
setIsModalOpen(true);
}}
uploadImage={uploadImage}
endFrameImage={isReferenceMode ? undefined : endFrameImage}
setEndFrameImage={isReferenceMode ? undefined : setEndFrameImage}
allowUploadEnd={!isReferenceMode && !!selectedModel?.endFrame}
showEndFrameSection={!isReferenceMode && !!selectedModel?.endFrame}
referenceVideos={isReferenceMode ? referenceVideos : undefined}
setReferenceVideos={isReferenceMode ? setReferenceVideos : undefined}
maxVideoCount={selectedModel?.maxReferenceVideos ?? 3}
maxVideoRefDuration={selectedModel?.maxVideoRefDuration ?? 15}
showVideoReferenceSection={isReferenceMode}
uploadVideo={uploadVideo}
referenceAudios={isReferenceMode ? referenceAudios : undefined}
setReferenceAudios={isReferenceMode ? setReferenceAudios : undefined}
maxAudioCount={selectedModel?.maxReferenceAudios ?? 2}
maxAudioRefDuration={selectedModel?.maxAudioRefDuration ?? 15}
uploadAudio={uploadAudio}
className={className}
/>
);
// Character button (seedance_2p0 only), reused in the fullscreen footer.
const characterButtonEl =
selectedModel?.id === "seedance_2p0" ? (
@@ -1042,21 +1239,12 @@ export const PromptBoxVideo = ({
return (
<>
<Modal
isOpen={isModalOpen}
onClose={() => {
setIsModalOpen(false);
setContent(null);
}}
>
{content}
</Modal>
{deck.fileInputs}
{deck.galleryModal}
<div className="relative z-20 flex flex-col gap-3">
{isImageRowVisible && renderImagePromptRow()}
<div
className={twMerge(
"glass relative w-full rounded-2xl p-4",
isImageRowVisible && "rounded-t-none",
isFocused
? "ring-1 ring-primary border-primary"
: "ring-1 ring-transparent",
@@ -1075,6 +1263,7 @@ export const PromptBoxVideo = ({
</div>
)}
<div className="relative flex justify-center gap-2">
{isReferenceMode ? renderReferenceDeck() : renderKeyframeCards()}
<div className="promptbox-resize-wrap relative flex-1 min-w-0">
{hasAnyMentionables ? (
<MentionTextarea
@@ -1243,18 +1432,14 @@ export const PromptBoxVideo = ({
disabled={!modelNeedsAnImageButNoneAreSelected}
>
<div>
<GenerateButton
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={undefined}
<GenerateIconButton
onClick={handleEnqueue}
disabled={!prompt.trim()}
loading={isEnqueueing}
credits={
credits != null ? credits * generationCount : credits
}
>
Generate
</GenerateButton>
/>
</div>
</Tooltip>
</div>
@@ -1314,34 +1499,6 @@ export const PromptBoxVideo = ({
});
}}
/>
<GalleryModal
isOpen={!!isGalleryModalOpen}
onClose={() => {
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
}}
mode="select"
selectedItemIds={selectedGalleryImages}
onSelectItem={(id) => {
setSelectedGalleryImages((prev) => (prev.includes(id) ? [] : [id]));
}}
maxSelections={1}
onUseSelected={(selectedItems: GalleryItem[]) => {
const item = selectedItems[0];
if (!item || !item.fullImage) return;
const referenceImage: RefImage = {
id: Math.random().toString(36).substring(7),
url: item.fullImage,
file: new File([], "library-image"),
mediaToken: item.id,
};
setReferenceImages([referenceImage]);
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
}}
onDownloadClicked={downloadFileFromUrl}
forceFilter="image"
/>
<PromptFullscreenModal
isOpen={isFullscreen}
onClose={closeFullscreen}
@@ -1354,7 +1511,9 @@ export const PromptBoxVideo = ({
{characterButtonEl}
</>
}
imagePromptRow={renderImagePromptRow("relative top-auto rounded-2xl")}
imagePromptRow={
isReferenceMode ? renderReferenceDeck(true) : renderKeyframeCards()
}
>
{hasAnyMentionables ? (
<MentionTextarea
@@ -49,7 +49,10 @@ export const PromptFullscreenModal = ({
// Focus the editor and drop the caret at the end once the overlay opens.
useEffect(() => {
if (!isOpen) return;
const id = window.setTimeout(() => focusEditorAtEnd(contentRef.current), 60);
const id = window.setTimeout(
() => focusEditorAtEnd(contentRef.current),
60,
);
return () => window.clearTimeout(id);
}, [isOpen]);
@@ -89,7 +92,7 @@ export const PromptFullscreenModal = ({
</span>
</div>
{imagePromptRow && <div className="shrink-0">{imagePromptRow}</div>}
<div className="flex shrink-0 items-center justify-between gap-2">
<div className="flex shrink-0 items-center justify-between gap-2 mt-1">
<div className="flex items-center gap-2">{footerControls}</div>
<Button onClick={onClose}>Done</Button>
</div>
@@ -1,6 +1,7 @@
import { useCallback, useRef, useState } from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faFolderOpen,
faImage,
faMusic,
faPlay,
@@ -19,6 +20,8 @@ export interface AudioReferenceRowProps {
maxAudioCount: number;
maxAudioRefDuration: number;
uploadAudio?: UploadMediaFn;
// Opens the caller's audio library picker (adds a "From library" button).
onPickAudioFromLibrary?: () => void;
// Whether the audio reference is required (remix/sample source).
audioRequired?: boolean;
// Optional single-image reference section (Seed Audio).
@@ -39,6 +42,7 @@ export function AudioReferenceRow({
maxAudioCount,
maxAudioRefDuration,
uploadAudio,
onPickAudioFromLibrary,
audioRequired = false,
imageSupported = false,
referenceImages = [],
@@ -197,15 +201,28 @@ export function AudioReferenceRow({
))}
{referenceAudios.length < maxAudioCount && (
<button
type="button"
onClick={() => audioInputRef.current?.click()}
disabled={isUploadingAudio}
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
>
<FontAwesomeIcon icon={faPlus} className="h-3 w-3" />
{isUploadingAudio ? "Uploading…" : "Add audio"}
</button>
<>
<button
type="button"
onClick={() => audioInputRef.current?.click()}
disabled={isUploadingAudio}
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
>
<FontAwesomeIcon icon={faPlus} className="h-3 w-3" />
{isUploadingAudio ? "Uploading…" : "Add audio"}
</button>
{onPickAudioFromLibrary && (
<button
type="button"
onClick={onPickAudioFromLibrary}
disabled={isUploadingAudio}
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
>
<FontAwesomeIcon icon={faFolderOpen} className="h-3 w-3" />
From library
</button>
)}
</>
)}
{imageSupported && (
@@ -0,0 +1,287 @@
import {
useCallback,
useEffect,
useRef,
useState,
type CSSProperties,
} from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faArrowUpFromBracket,
faImages,
faPlay,
faSpinnerThird,
faStop,
faXmark,
} from "@fortawesome/pro-solid-svg-icons";
import { faMusic, faVideo } from "@fortawesome/pro-regular-svg-icons";
import { Modal } from "@storyteller/ui-modal";
import { twMerge } from "tailwind-merge";
import { DeckAddAction, DeckItem } from "./deckTypes";
/**
* One-shot pop-in used when a card is added to the deck. Injected inline so
* the lib doesn't depend on app Tailwind configs defining the keyframes.
*/
export const DECK_KEYFRAMES = `
@keyframes deck-pop {
from { opacity: 0; transform: scale(0.6); }
to { opacity: 1; transform: scale(1); }
}
`;
export const DeckStyles = () => <style>{DECK_KEYFRAMES}</style>;
interface DeckCardProps {
item: DeckItem;
onRemove?: () => void;
onClick?: () => void;
/** Force-hide hover chrome (remove button) — used while dragging. */
hideHoverChrome?: boolean;
/** Skip the pop-in animation (collapsed fan re-renders shouldn't pop). */
animateIn?: boolean;
className?: string;
style?: CSSProperties;
}
/**
* A single reference card face: image/video thumbnail or audio play tile,
* duration badge, uploading spinner, hover remove and hover name label.
*/
export const DeckCard = ({
item,
onRemove,
onClick,
hideHoverChrome,
animateIn,
className,
style,
}: DeckCardProps) => {
const audioRef = useRef<HTMLAudioElement | null>(null);
const [isPlaying, setIsPlaying] = useState(false);
const stopAudio = useCallback(() => {
if (audioRef.current) {
audioRef.current.pause();
audioRef.current = null;
}
setIsPlaying(false);
}, []);
const handleToggleAudio = useCallback(() => {
if (isPlaying) {
stopAudio();
} else if (item.url) {
const el = new Audio(item.url);
el.volume = 0.2;
audioRef.current = el;
el.onended = () => setIsPlaying(false);
el.play();
setIsPlaying(true);
}
}, [isPlaying, item.url, stopAudio]);
useEffect(() => stopAudio, [stopAudio]);
const handleClick = () => {
if (item.uploading) return;
if (item.kind === "audio") {
handleToggleAudio();
} else {
onClick?.();
}
};
return (
<div
className={twMerge(
"glass group relative aspect-square w-14 shrink-0 overflow-hidden rounded-lg border-2 border-white/30 transition-all duration-200",
!item.uploading &&
"cursor-pointer hover:border-white/80 hover:cursor-zoom-in",
item.kind === "audio" && "hover:cursor-pointer",
animateIn && "animate-[deck-pop_180ms_ease-out]",
className,
)}
style={style}
onClick={handleClick}
>
{item.kind === "image" && item.url && (
<img
src={item.url}
alt={item.name}
loading="lazy"
className={twMerge(
"h-full w-full object-cover",
item.uploading && "blur-sm",
)}
/>
)}
{item.kind === "video" && item.url && (
<video
src={item.url}
muted
preload="metadata"
className={twMerge(
"h-full w-full object-cover",
item.uploading && "blur-sm",
)}
/>
)}
{item.kind === "audio" && (
<div className="flex h-full w-full items-center justify-center">
<FontAwesomeIcon
icon={isPlaying ? faStop : faPlay}
className={twMerge(
"h-5 w-5 transition-colors",
isPlaying
? "text-red-400"
: "text-base-fg/60 group-hover:text-base-fg",
)}
/>
</div>
)}
{item.kind === "video" && (
<div className="pointer-events-none absolute left-[3px] top-[3px] flex h-4 w-4 items-center justify-center rounded bg-black/60 text-white">
<FontAwesomeIcon icon={faVideo} className="h-2.5 w-2.5" />
</div>
)}
{item.kind === "audio" && (
<div className="pointer-events-none absolute left-[3px] top-[3px] flex h-4 w-4 items-center justify-center rounded bg-black/60 text-white">
<FontAwesomeIcon icon={faMusic} className="h-2.5 w-2.5" />
</div>
)}
{item.duration != null && (
<div className="pointer-events-none absolute bottom-0 left-0 right-0 flex items-center justify-center bg-black/70 py-0.5 text-[10px] font-bold text-white">
{item.duration}s
</div>
)}
{item.uploading && (
<div className="absolute inset-0 flex items-center justify-center bg-black/20">
<FontAwesomeIcon
icon={faSpinnerThird}
className="h-6 w-6 animate-spin text-white"
/>
</div>
)}
{onRemove && !item.uploading && !hideHoverChrome && (
<button
onClick={(e) => {
e.stopPropagation();
stopAudio();
onRemove();
}}
onMouseDown={(e) => e.stopPropagation()}
onPointerDown={(e) => e.stopPropagation()}
className="absolute right-[2px] top-[2px] flex h-5 w-5 cursor-pointer items-center justify-center rounded-full bg-black/50 text-white opacity-0 backdrop-blur-md transition-colors hover:bg-red/70 group-hover:opacity-100"
>
<FontAwesomeIcon icon={faXmark} className="h-2.5 w-2.5" />
</button>
)}
</div>
);
};
/**
* Uniform add-menu body: ghost rows grouped by media type with small
* uppercase section headers and separators. `groupHints` puts a compact
* limits readout on the right of each header (e.g. "2/9", "1/3 · 8/15s").
*/
export const DeckAddMenu = ({
actions,
groupHints,
}: {
actions: DeckAddAction[];
groupHints?: Record<string, string>;
}) => {
const groups: { name?: string; actions: DeckAddAction[] }[] = [];
for (const action of actions) {
const last = groups[groups.length - 1];
if (last && last.name === action.group) {
last.actions.push(action);
} else {
groups.push({ name: action.group, actions: [action] });
}
}
const showHeaders =
groups.some((g) => g.name) && (groups.length > 1 || !!groupHints);
return (
<div className="flex w-48 flex-col">
{groups.map((group, groupIndex) => (
<div
key={group.name ?? groupIndex}
className={
groupIndex > 0 ? "mt-1 border-t border-white/10 pt-1" : undefined
}
>
{showHeaders && group.name && (
<div className="flex items-center justify-between gap-3 px-2.5 pb-0.5 pt-1">
<span className="text-[10px] font-semibold uppercase tracking-wider text-base-fg/40">
{group.name}
</span>
{groupHints?.[group.name] && (
<span className="text-[10px] font-medium tabular-nums text-base-fg/40">
{groupHints[group.name]}
</span>
)}
</div>
)}
{group.actions.map((action) => (
<button
key={action.key}
type="button"
onClick={action.onSelect}
className="flex w-full items-center gap-2.5 rounded-md px-2.5 py-1.5 text-left text-[13px] font-medium text-base-fg transition-colors hover:bg-white/10"
>
<FontAwesomeIcon
icon={
action.icon ??
(action.key.startsWith("upload")
? faArrowUpFromBracket
: faImages)
}
className="h-3.5 w-3.5 opacity-60"
/>
{action.label}
</button>
))}
</div>
))}
</div>
);
};
interface DeckPreviewModalProps {
item: DeckItem | null;
onClose: () => void;
}
/** Darkened-backdrop fullscreen preview for a clicked deck card. */
export const DeckPreviewModal = ({ item, onClose }: DeckPreviewModalProps) => (
<Modal
isOpen={item !== null}
onClose={onClose}
backdropClassName="!bg-black/80"
className="h-fit w-fit max-w-none border-0 bg-transparent p-0 shadow-none"
>
{item &&
(item.kind === "video" ? (
<video
src={item.previewUrl ?? item.url}
controls
autoPlay
className="max-h-[90vh] max-w-[90vw] rounded-lg object-contain"
/>
) : (
<img
src={item.previewUrl ?? item.url}
alt={item.name}
className="max-h-[90vh] max-w-[90vw] rounded-lg object-contain"
/>
))}
</Modal>
);
@@ -0,0 +1,172 @@
import { useState } from "react";
import { Tooltip } from "@storyteller/ui-tooltip";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faArrowRightArrowLeft,
faPlus,
} from "@fortawesome/pro-solid-svg-icons";
import { twMerge } from "tailwind-merge";
import { DeckAddAction, DeckItem } from "./deckTypes";
import {
DeckAddMenu,
DeckCard,
DeckPreviewModal,
DeckStyles,
} from "./DeckCard";
interface KeyframeCardsProps {
firstFrame?: DeckItem;
lastFrame?: DeckItem;
/** Whether the model supports an ending keyframe. */
showLastFrame: boolean;
onFirstAddActions: DeckAddAction[];
onLastAddActions?: DeckAddAction[];
onRemoveFirst: () => void;
onRemoveLast?: () => void;
/** Exchange first/last frames; only shown when both are set. */
onSwap?: () => void;
className?: string;
}
/**
* Dreamina-style keyframe widget: two slightly tilted "First frame" /
* "Last frame" card slots with a swap glyph between them, sitting left of
* the prompt text inside the promptbox.
*/
export const KeyframeCards = ({
firstFrame,
lastFrame,
showLastFrame,
onFirstAddActions,
onLastAddActions = [],
onRemoveFirst,
onRemoveLast,
onSwap,
className,
}: KeyframeCardsProps) => {
const [previewItem, setPreviewItem] = useState<DeckItem | null>(null);
return (
<div
className={twMerge(
"relative flex shrink-0 items-center self-center",
className,
)}
>
<DeckStyles />
<KeyframeSlot
item={firstFrame}
label="First frame"
tiltClass={showLastFrame ? "-rotate-6" : ""}
addActions={onFirstAddActions}
onRemove={onRemoveFirst}
onPreview={setPreviewItem}
/>
{showLastFrame && (
<>
<div className="z-10 -mx-1.5 flex items-center justify-center">
{firstFrame && lastFrame && onSwap ? (
<button
type="button"
onClick={onSwap}
title="Swap frames"
className="flex h-5 w-5 items-center justify-center rounded-full border border-white/20 bg-black/60 text-[10px] text-white shadow backdrop-blur-md transition-all hover:scale-110 hover:bg-black/80"
>
<FontAwesomeIcon icon={faArrowRightArrowLeft} />
</button>
) : (
<div className="pointer-events-none flex h-5 w-5 items-center justify-center rounded-full border border-white/10 bg-black/40 text-[10px] text-white/50 backdrop-blur-md">
<FontAwesomeIcon icon={faArrowRightArrowLeft} />
</div>
)}
</div>
<KeyframeSlot
item={lastFrame}
label="Last frame"
tiltClass="rotate-6"
addActions={onLastAddActions}
onRemove={onRemoveLast}
onPreview={setPreviewItem}
/>
</>
)}
<DeckPreviewModal
item={previewItem}
onClose={() => setPreviewItem(null)}
/>
</div>
);
};
const KeyframeSlot = ({
item,
label,
tiltClass,
addActions,
onRemove,
onPreview,
}: {
item?: DeckItem;
label: string;
tiltClass: string;
addActions: DeckAddAction[];
onRemove?: () => void;
onPreview: (item: DeckItem) => void;
}) => {
const enabledActions = addActions.filter((a) => !a.disabled);
if (item) {
return (
<div className="flex flex-col items-center gap-0.5">
<DeckCard
item={item}
onRemove={item.uploading ? undefined : onRemove}
onClick={() => onPreview(item)}
className={twMerge(tiltClass, "hover:z-10 hover:rotate-0")}
/>
<span className="text-[9px] font-medium leading-none text-base-fg/60">
{label}
</span>
</div>
);
}
const emptyCard = (
<button
type="button"
onClick={() => enabledActions[0]?.onSelect()}
className={twMerge(
"glass flex aspect-square w-14 flex-col items-center justify-center gap-0.5 rounded-lg border-2 border-dashed border-black/5 bg-ui-controls/40 text-base-fg transition-all duration-200 hover:z-10 hover:rotate-0 hover:scale-105 hover:bg-ui-controls/60 dark:border-white/25",
tiltClass,
)}
>
<FontAwesomeIcon icon={faPlus} className="text-lg opacity-80" />
<span className="px-0.5 text-center text-[8px] font-medium leading-tight opacity-70">
{label}
</span>
</button>
);
if (enabledActions.length === 0) {
return emptyCard;
}
return enabledActions.length > 1 ? (
<Tooltip
interactive={true}
position="top"
delay={100}
className="-mb-0.5 border border-ui-controls-border bg-ui-controls p-1.5 text-base-fg"
closeOnClick={true}
content={<DeckAddMenu actions={enabledActions} />}
>
{emptyCard}
</Tooltip>
) : (
emptyCard
);
};
@@ -0,0 +1,506 @@
import {
useCallback,
useLayoutEffect,
useMemo,
useRef,
useState,
} from "react";
import { Tooltip } from "@storyteller/ui-tooltip";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faPlus, faTrashAlt } from "@fortawesome/pro-solid-svg-icons";
import { twMerge } from "tailwind-merge";
import {
DndContext,
closestCenter,
KeyboardSensor,
PointerSensor,
useSensor,
useSensors,
DragEndEvent,
} from "@dnd-kit/core";
import {
SortableContext,
sortableKeyboardCoordinates,
rectSortingStrategy,
useSortable,
} from "@dnd-kit/sortable";
import { CSS } from "@dnd-kit/utilities";
import { DeckAddAction, DeckItem } from "./deckTypes";
import {
DeckAddMenu,
DeckCard,
DeckPreviewModal,
DeckStyles,
} from "./DeckCard";
/** Collapsed-fan tilt/offset per stacked card, front to back. */
const FAN_TRANSFORMS = [
"rotate(-8deg)",
"translateX(12px) rotate(2deg)",
"translateX(24px) rotate(9deg)",
];
const CLOSE_DELAY_MS = 150;
// Expanding is delayed so the pointer can travel across the cards to the
// circular "+" without the panel opening over it mid-way.
const OPEN_DELAY_MS = 170;
interface ReferenceDeckProps {
items: DeckItem[];
onRemove: (id: string) => void;
/** Reorder within the image group; caller applies arrayMove to its image array. */
onReorderImages?: (fromIndex: number, toIndex: number) => void;
/** Entries for the "+" add menu. Empty → no add affordance. */
addActions?: DeckAddAction[];
/** Compact limits per menu group, shown right of the group header —
* e.g. { image: "2/9", video: "1/3 · 8/15s" }. */
addMenuGroupHints?: Record<string, string>;
/** Direct click on the reference card / circular "+". Defaults to the
* first enabled add action; pass the combined any-file picker to accept
* every supported media type in one dialog. */
onAddClick?: () => void;
/** Whether more refs can be added (caller computes from per-type limits). */
canAdd: boolean;
/** Clears every attached ref; shown in the expanded panel when 2+ items. */
onClearAll?: () => void;
/** Label on the empty placeholder card. */
emptyLabel?: string;
/** Render fully expanded and in-flow (fullscreen prompt modal). */
alwaysExpanded?: boolean;
/** Max cards in the collapsed fan before the +N badge. */
maxCollapsed?: number;
className?: string;
}
/**
* Dreamina-style reference deck: a fanned stack of tilted thumbnail cards
* that expands on hover into a wrap/scroll panel with drag-reorder, hover
* remove, name labels, and a fullscreen preview. Scales to 50 refs — the
* collapsed fan mounts at most `maxCollapsed` media elements.
*/
export const ReferenceDeck = ({
items,
onRemove,
onReorderImages,
addActions = [],
addMenuGroupHints,
onAddClick,
canAdd,
onClearAll,
emptyLabel = "Reference",
alwaysExpanded,
maxCollapsed = 3,
className,
}: ReferenceDeckProps) => {
const [hovered, setHovered] = useState(false);
const [isDragging, setIsDragging] = useState(false);
const [addMenuOpen, setAddMenuOpen] = useState(false);
const [previewItem, setPreviewItem] = useState<DeckItem | null>(null);
const closeTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const openTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const rootRef = useRef<HTMLDivElement>(null);
const [panelMaxWidth, setPanelMaxWidth] = useState<number>();
// Hold the panel open while dragging or while the add menu is up —
// collapsing mid-drag would unmount the sortable context and abort it.
const expanded = alwaysExpanded || hovered || isDragging || addMenuOpen;
// The panel never scrolls: it fits every ref, growing rightward up to the
// far edge of the promptbox (the enclosing .glass container), then wraps
// into additional rows and grows taller/upward. The panel's absolute
// positioning context is the small deck root, so measure the promptbox.
useLayoutEffect(() => {
if (!expanded || alwaysExpanded) return;
const root = rootRef.current;
const box = root?.closest(".glass");
if (root && box) {
const available =
box.getBoundingClientRect().right -
root.getBoundingClientRect().left -
12;
setPanelMaxWidth(available > 120 ? available : undefined);
}
}, [expanded, alwaysExpanded]);
const enabledActions = useMemo(
() => addActions.filter((a) => !a.disabled),
[addActions],
);
const imageItems = useMemo(
() => items.filter((i) => i.kind === "image" && !i.uploading),
[items],
);
const allowReorder = !!onReorderImages && imageItems.length > 1;
const sensors = useSensors(
useSensor(PointerSensor, { activationConstraint: { distance: 6 } }),
useSensor(KeyboardSensor, {
coordinateGetter: sortableKeyboardCoordinates,
}),
);
const clearOpenTimer = () => {
if (openTimer.current) {
clearTimeout(openTimer.current);
openTimer.current = null;
}
};
const clearCloseTimer = () => {
if (closeTimer.current) {
clearTimeout(closeTimer.current);
closeTimer.current = null;
}
};
// Opening is armed by the card stack only (delayed, so the pointer can
// reach the circular "+" first). Closing is scoped to the deck ROOT, which
// contains both the stack and the expanded panel: when the panel opens
// over the cursor the hit target swaps to it without leaving the root, so
// no close fires and the open/close flicker loop can't happen.
const handleStackEnter = () => {
clearOpenTimer();
openTimer.current = setTimeout(() => setHovered(true), OPEN_DELAY_MS);
};
const handleStackLeave = () => {
clearOpenTimer();
};
const handleRootEnter = () => {
clearCloseTimer();
};
const handleRootLeave = () => {
clearOpenTimer();
clearCloseTimer();
closeTimer.current = setTimeout(() => setHovered(false), CLOSE_DELAY_MS);
};
const handleDragEnd = useCallback(
(event: DragEndEvent) => {
setIsDragging(false);
const { active, over } = event;
if (!over || active.id === over.id) return;
const fromIndex = imageItems.findIndex((i) => i.id === active.id);
const toIndex = imageItems.findIndex((i) => i.id === over.id);
if (fromIndex === -1 || toIndex === -1) return;
onReorderImages?.(fromIndex, toIndex);
},
[imageItems, onReorderImages],
);
// Clicking the + card / reference card opens the combined file picker when
// the caller provides one (routes by MIME type), else the primary action.
// Hovering reveals the full menu when there are more options.
const handleAddClick = () => {
if (onAddClick) {
onAddClick();
} else {
enabledActions[0]?.onSelect();
}
};
// holdPanel: keep the expanded panel open while this menu is up (used by
// the + card inside the panel). The collapsed stack's circular + must NOT
// hold the panel open — expanding would cover its own trigger.
const addMenu = (trigger: React.ReactElement, holdPanel = true) =>
enabledActions.length > 1 ? (
<Tooltip
interactive={true}
position="top"
delay={100}
className="-mb-0.5 border border-ui-controls-border bg-ui-controls p-1.5 text-base-fg"
closeOnClick={true}
onOpenChange={holdPanel ? setAddMenuOpen : undefined}
content={
<DeckAddMenu
actions={enabledActions}
groupHints={addMenuGroupHints}
/>
}
>
{trigger}
</Tooltip>
) : (
trigger
);
if (items.length === 0 && enabledActions.length === 0) {
return null;
}
// ---- Empty state: single tilted dashed placeholder card ----
if (items.length === 0) {
return (
<div
className={twMerge(
"relative flex shrink-0 items-center self-center",
className,
)}
>
<DeckStyles />
{addMenu(
<button
type="button"
onClick={handleAddClick}
className="glass flex aspect-square w-14 -rotate-6 flex-col items-center justify-center gap-0.5 rounded-lg border-2 border-dashed border-black/5 bg-ui-controls/40 text-base-fg transition-all duration-200 hover:rotate-0 hover:scale-105 hover:bg-ui-controls/60 dark:border-white/25"
>
<FontAwesomeIcon icon={faPlus} className="text-lg opacity-80" />
<span className="text-[9px] font-medium leading-none opacity-70">
{emptyLabel}
</span>
</button>,
)}
</div>
);
}
const collapsedItems = items.slice(0, maxCollapsed);
const overflowCount = items.length - collapsedItems.length;
const expandedPanel = (
<div
className={twMerge(
"glass w-max rounded-xl border border-white/10 p-2 shadow-2xl",
alwaysExpanded
? "relative"
: twMerge(
"absolute bottom-0 left-0 z-40 origin-bottom-left transition-all duration-200 ease-out",
expanded
? "visible translate-y-0 scale-100 opacity-100"
: "pointer-events-none invisible translate-y-1 scale-95 opacity-0 [transition:opacity_.2s,transform_.2s,visibility_0s_.2s]",
),
)}
style={
!alwaysExpanded && panelMaxWidth
? { maxWidth: `${panelMaxWidth}px` }
: undefined
}
>
{onClearAll && items.length > 1 && (
<div className="mb-1.5 flex items-center justify-between px-0.5">
<span className="text-[11px] font-medium text-base-fg/60">
{items.length} reference{items.length !== 1 ? "s" : ""}
</span>
<button
type="button"
onClick={onClearAll}
className="flex items-center gap-1 rounded px-1 py-0.5 text-[11px] text-base-fg/60 transition-colors hover:text-red-400"
>
<FontAwesomeIcon icon={faTrashAlt} className="h-2.5 w-2.5" />
Clear all
</button>
</div>
)}
<div className="flex flex-wrap gap-2">
<DndContext
sensors={sensors}
collisionDetection={closestCenter}
onDragStart={() => setIsDragging(true)}
onDragCancel={() => setIsDragging(false)}
onDragEnd={handleDragEnd}
>
<SortableContext
items={imageItems.map((i) => i.id)}
strategy={rectSortingStrategy}
>
{items.map((item) =>
allowReorder && item.kind === "image" && !item.uploading ? (
<SortableDeckCard
key={item.id}
item={item}
hideHoverChrome={isDragging}
onRemove={() => onRemove(item.id)}
onClick={() => setPreviewItem(item)}
/>
) : (
<Tooltip
key={item.id}
content={item.name}
position="top"
delay={150}
disabled={isDragging || item.uploading}
>
<DeckCard
item={item}
animateIn
hideHoverChrome={isDragging}
onRemove={
item.uploading ? undefined : () => onRemove(item.id)
}
onClick={() => setPreviewItem(item)}
/>
</Tooltip>
),
)}
</SortableContext>
</DndContext>
{canAdd &&
enabledActions.length > 0 &&
addMenu(
<button
type="button"
onClick={handleAddClick}
className="glass flex aspect-square w-14 shrink-0 items-center justify-center rounded-lg border-2 border-dashed border-black/5 bg-ui-controls/40 text-base-fg transition-all hover:bg-ui-controls/60 dark:border-white/25"
>
<FontAwesomeIcon icon={faPlus} className="text-xl opacity-80" />
</button>,
)}
</div>
</div>
);
if (alwaysExpanded) {
return (
<div className={twMerge("relative flex shrink-0 items-center", className)}>
<DeckStyles />
{expandedPanel}
<DeckPreviewModal
item={previewItem}
onClose={() => setPreviewItem(null)}
/>
</div>
);
}
return (
<div
ref={rootRef}
className={twMerge(
"relative flex shrink-0 items-center self-center",
className,
)}
onMouseEnter={handleRootEnter}
onMouseLeave={handleRootLeave}
>
<DeckStyles />
{/* Collapsed fan — fixed footprint so the textarea never reflows.
Only the card stack expands the panel on hover; the circular +
in front has its own hover menu. */}
<div
className={twMerge(
"relative h-14 w-[92px] transition-opacity duration-150",
expanded && "pointer-events-none opacity-0",
)}
>
<div
className="absolute inset-y-0 left-0"
style={{
// Hover zone hugs the actual card stack — with one card the
// dead space next to it must not trigger the expand.
width: `${(collapsedItems.length - 1) * 12 + 56}px`,
}}
onMouseEnter={handleStackEnter}
onMouseLeave={handleStackLeave}
>
{collapsedItems.map((item, index) => (
<DeckCard
key={item.id}
item={item}
className="absolute left-0 top-0 shadow-md"
style={{
transform: FAN_TRANSFORMS[index] ?? FAN_TRANSFORMS[2],
zIndex: collapsedItems.length - index,
}}
/>
))}
{overflowCount > 0 && (
<div className="pointer-events-none absolute -right-1 -top-1 z-10 rounded-full bg-primary px-1.5 py-0.5 text-[10px] font-bold text-white shadow">
+{overflowCount}
</div>
)}
</div>
{canAdd && enabledActions.length > 0 && (
// The Tooltip wraps its trigger in a plain `relative` div, so the
// absolute anchor must live on this wrapper — putting it on the
// button would position it against the tooltip wrapper instead of
// the deck.
<div
className="absolute z-20"
style={{
left: `${(collapsedItems.length - 1) * 12 + 56 - 12}px`,
bottom: "-4px",
}}
>
{addMenu(
<button
type="button"
onClick={handleAddClick}
className="flex h-6 w-6 items-center justify-center rounded-full border border-white/15 bg-ui-controls text-xs text-base-fg shadow-md transition-all hover:scale-110 hover:brightness-125"
>
<FontAwesomeIcon icon={faPlus} />
</button>,
false,
)}
</div>
)}
</div>
{expandedPanel}
<DeckPreviewModal
item={previewItem}
onClose={() => setPreviewItem(null)}
/>
</div>
);
};
const SortableDeckCard = ({
item,
onRemove,
onClick,
hideHoverChrome,
}: {
item: DeckItem;
onRemove: () => void;
onClick: () => void;
hideHoverChrome?: boolean;
}) => {
const {
attributes,
listeners,
setNodeRef,
transform,
transition,
isDragging,
} = useSortable({ id: item.id });
return (
<div
ref={setNodeRef}
style={{
transform: CSS.Transform.toString(transform),
transition,
zIndex: isDragging ? 9999 : undefined,
}}
{...attributes}
{...listeners}
className={isDragging ? "opacity-50" : undefined}
>
<Tooltip
content={item.name}
position="top"
delay={150}
disabled={hideHoverChrome || isDragging}
>
<DeckCard
item={item}
animateIn
hideHoverChrome={hideHoverChrome || isDragging}
onRemove={onRemove}
onClick={onClick}
className={
isDragging
? "cursor-grabbing hover:cursor-grabbing"
: "cursor-grab hover:cursor-grab active:cursor-grabbing"
}
/>
</Tooltip>
</div>
);
};
@@ -0,0 +1,35 @@
import type { IconDefinition } from "@fortawesome/fontawesome-svg-core";
export type DeckItemKind = "image" | "video" | "audio";
/**
* Neutral reference item consumed by the deck UI. Callers adapt their own
* ref shapes (desktop promptStore RefImage/RefVideo/RefAudio, webapp
* RefImage with fullUrl) into this.
*/
export interface DeckItem {
id: string;
kind: DeckItemKind;
/** Thumbnail / object URL. Absent for audio. */
url?: string;
/** Full-res URL for the preview modal; falls back to `url`. */
previewUrl?: string;
/** Hover label — "Image 1", "Video 2", "Audio 1". Numbering owned by caller. */
name: string;
/** Seconds badge for video/audio cards. */
duration?: number;
/** True while its upload is in flight — blurred thumb + spinner overlay. */
uploading?: boolean;
}
/** One entry in a deck "+" add menu (upload, pick from library, ...). */
export interface DeckAddAction {
key: string;
label: string;
icon?: IconDefinition;
onSelect: () => void;
disabled?: boolean;
/** Menu section ("image" | "video" | "audio"); shown as a header when the
* menu spans multiple groups. */
group?: string;
}
@@ -0,0 +1,736 @@
import { useEffect, useRef, useState, type ReactNode } from "react";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import { downloadFileFromUrl, type UploadMediaFn } from "@storyteller/api";
import { toast } from "@storyteller/ui-toaster";
import { UploaderStates } from "@storyteller/common";
/** Minimal structural shapes so both apps' ref types fit. */
export interface DeckRefLike {
id: string;
url: string;
}
export interface DeckMediaRefLike extends DeckRefLike {
duration: number;
}
/** An in-flight upload; `previewUrl` is a memoized object URL. */
export interface DeckUploadEntry {
id: string;
file: File;
previewUrl: string;
}
export interface UseDeckMediaOptions<
TImage extends DeckRefLike,
TVideo extends DeckMediaRefLike,
TAudio extends DeckMediaRefLike,
> {
referenceImages: TImage[];
setReferenceImages: (images: TImage[]) => void;
maxImages: number;
setEndFrameImage?: (image?: TImage) => void;
referenceVideos?: TVideo[];
setReferenceVideos?: (videos: TVideo[]) => void;
maxVideos?: number;
maxVideoTotalSec?: number;
referenceAudios?: TAudio[];
setReferenceAudios?: (audios: TAudio[]) => void;
maxAudios?: number;
maxAudioTotalSec?: number;
uploadImage?: UploadMediaFn;
uploadVideo?: UploadMediaFn;
uploadAudio?: UploadMediaFn;
/**
* When true the hook owns a target-aware GalleryModal (desktop). When
* false the caller keeps its own library pickers (webapp) and only the
* upload paths are used.
*/
ownGalleryModal?: boolean;
}
const randomId = () => Math.random().toString(36).substring(7);
const randomTitle = (prefix: string) =>
`${prefix}-${Math.random().toString(36).substring(2, 15)}`;
const getVideoDurationFromSrc = (src: string): Promise<number> =>
new Promise((resolve) => {
const video = document.createElement("video");
video.preload = "metadata";
video.onloadedmetadata = () => resolve(Math.round(video.duration));
video.onerror = () => resolve(0);
video.src = src;
});
const getVideoDuration = (file: File): Promise<number> => {
const src = URL.createObjectURL(file);
return getVideoDurationFromSrc(src).finally(() => URL.revokeObjectURL(src));
};
const getAudioDurationFromSrc = (src: string): Promise<number> =>
new Promise((resolve) => {
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.onloadedmetadata = () => resolve(Math.round(audio.duration));
audio.onerror = () => resolve(0);
audio.src = src;
});
const getAudioDuration = (file: File): Promise<number> =>
new Promise((resolve) => {
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.onloadedmetadata = () => {
URL.revokeObjectURL(audio.src);
resolve(Math.round(audio.duration));
};
audio.onerror = () => {
URL.revokeObjectURL(audio.src);
resolve(0);
};
audio.src = URL.createObjectURL(file);
});
/**
* Headless upload/limits/library state machine for the reference deck.
* Extracted from the legacy ImagePromptRow band so PromptBoxImage,
* PromptBoxVideo, and the webapp PromptBox share one implementation.
*/
export function useDeckMedia<
TImage extends DeckRefLike,
TVideo extends DeckMediaRefLike,
TAudio extends DeckMediaRefLike,
>({
referenceImages,
setReferenceImages,
maxImages,
setEndFrameImage,
referenceVideos = [],
setReferenceVideos,
maxVideos = 3,
maxVideoTotalSec = 15,
referenceAudios = [],
setReferenceAudios,
maxAudios = 2,
maxAudioTotalSec = 15,
uploadImage,
uploadVideo,
uploadAudio,
ownGalleryModal,
}: UseDeckMediaOptions<TImage, TVideo, TAudio>) {
const fileInputRef = useRef<HTMLInputElement>(null);
const videoFileInputRef = useRef<HTMLInputElement>(null);
const audioFileInputRef = useRef<HTMLInputElement>(null);
const anyFileInputRef = useRef<HTMLInputElement>(null);
const imageTargetRef = useRef<"start" | "end">("start");
const [uploadingImages, setUploadingImages] = useState<DeckUploadEntry[]>([]);
const [uploadingEnd, setUploadingEnd] = useState<DeckUploadEntry | null>(
null,
);
const [uploadingVideo, setUploadingVideo] = useState<DeckUploadEntry | null>(
null,
);
const [uploadingAudio, setUploadingAudio] = useState<DeckUploadEntry | null>(
null,
);
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
const [galleryTarget, setGalleryTarget] = useState<
"start" | "end" | "video" | "audio"
>("start");
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[],
);
const [isProcessingGallery, setIsProcessingGallery] = useState(false);
// Async upload completions must append to the freshest committed arrays,
// not the arrays captured when the upload started.
const referenceImagesRef = useRef(referenceImages);
useEffect(() => {
referenceImagesRef.current = referenceImages;
}, [referenceImages]);
const referenceAudiosRef = useRef(referenceAudios);
useEffect(() => {
referenceAudiosRef.current = referenceAudios;
}, [referenceAudios]);
// The hook only ever commits `{ id, url, file, mediaToken, duration? }`,
// which is structurally assignable to both apps' ref types (desktop
// promptStore refs match exactly; the webapp's extra fields are optional).
const asImage = (img: {
id: string;
url: string;
file: File;
mediaToken: string;
}) => img as unknown as TImage;
const asVideo = (video: {
id: string;
url: string;
file: File;
mediaToken: string;
duration: number;
}) => video as unknown as TVideo;
const asAudio = (audio: {
id: string;
url: string;
file: File;
mediaToken: string;
duration: number;
}) => audio as unknown as TAudio;
const makeEntry = (file: File): DeckUploadEntry => ({
id: randomId(),
file,
previewUrl: URL.createObjectURL(file),
});
const openImageUpload = () => {
imageTargetRef.current = "start";
fileInputRef.current?.click();
};
const openEndUpload = () => {
imageTargetRef.current = "end";
fileInputRef.current?.click();
};
const openVideoUpload = () => videoFileInputRef.current?.click();
const openAudioUpload = () => audioFileInputRef.current?.click();
const openAnyUpload = () => anyFileInputRef.current?.click();
const openGallery = (target: "start" | "end" | "video" | "audio") => {
setGalleryTarget(target);
setIsGalleryModalOpen(true);
};
const processImageFiles = (
files: File[],
uploadTarget: "start" | "end",
) => {
const currentCount = referenceImages.length + uploadingImages.length;
const availableSlots = Math.max(0, maxImages - currentCount);
if (availableSlots <= 0 && uploadTarget !== "end") {
return;
}
const filesToProcess =
uploadTarget === "end"
? files.slice(0, 1)
: files.slice(0, availableSlots);
filesToProcess.forEach((file) => {
const entry = makeEntry(file);
if (uploadTarget === "end") {
setUploadingEnd(entry);
} else {
setUploadingImages((prev) => [...prev, entry]);
}
const finishEntry = () => {
URL.revokeObjectURL(entry.previewUrl);
if (uploadTarget === "end") {
setUploadingEnd(null);
} else {
setUploadingImages((prev) => prev.filter((e) => e.id !== entry.id));
}
};
const commit = (url: string, mediaToken: string) => {
const referenceImage = asImage({
id: randomId(),
url,
file,
mediaToken,
});
finishEntry();
if (uploadTarget === "end") {
setEndFrameImage?.(referenceImage);
} else {
setReferenceImages([...referenceImagesRef.current, referenceImage]);
}
};
const reader = new FileReader();
reader.onloadend = async () => {
if (uploadImage) {
await uploadImage({
title: randomTitle("reference-image"),
assetFile: file,
progressCallback: (newState) => {
if (newState.status === UploaderStates.success && newState.data) {
commit(reader.result as string, newState.data);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
finishEntry();
}
},
});
} else {
commit(reader.result as string, "");
}
};
reader.readAsDataURL(file);
});
};
const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
const files = Array.from(event.target.files || []);
if (fileInputRef.current) fileInputRef.current.value = "";
if (files.length === 0) return;
processImageFiles(files, imageTargetRef.current);
};
const processVideoFiles = async (files: File[]) => {
// Snapshot the committed state at call time so removes that happened
// before this call are respected (don't re-read a stale ref).
const baseVideos = [...referenceVideos];
const availableSlots = Math.max(0, maxVideos - baseVideos.length);
if (availableSlots <= 0) {
toast.error(`Max ${maxVideos} videos / ${maxVideoTotalSec}s total`, {
id: "video-ref-limit",
});
return;
}
const filesToProcess = files.slice(0, availableSlots);
let committed = baseVideos;
for (const file of filesToProcess) {
const duration = await getVideoDuration(file);
const currentTotal = committed.reduce((sum, v) => sum + v.duration, 0);
if (currentTotal + duration > maxVideoTotalSec) {
toast.error(`Total video duration cannot exceed ${maxVideoTotalSec}s`, {
id: "video-ref-limit",
});
break;
}
const entry = makeEntry(file);
setUploadingVideo(entry);
const commit = (mediaToken: string) => {
// Reuse the entry's object URL as the committed thumbnail so it
// stays alive exactly as long as the ref does.
const refVideo = asVideo({
id: randomId(),
url: entry.previewUrl,
file,
mediaToken,
duration,
});
setUploadingVideo(null);
committed = [...committed, refVideo];
setReferenceVideos?.(committed);
};
if (uploadVideo) {
await uploadVideo({
title: randomTitle("reference-video"),
assetFile: file,
progressCallback: (newState) => {
if (newState.status === UploaderStates.success && newState.data) {
commit(newState.data);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
URL.revokeObjectURL(entry.previewUrl);
setUploadingVideo(null);
toast.error("Failed to upload video. Please upload an MP4 file.");
}
},
});
} else {
commit("");
}
}
};
const handleVideoFileUpload = async (
event: React.ChangeEvent<HTMLInputElement>,
) => {
const files = Array.from(event.target.files || []);
if (videoFileInputRef.current) videoFileInputRef.current.value = "";
if (files.length === 0) return;
await processVideoFiles(files);
};
const processAudioFiles = async (files: File[]) => {
const availableSlots = Math.max(0, maxAudios - referenceAudios.length);
if (availableSlots <= 0) {
return;
}
const filesToProcess = files.slice(0, availableSlots);
for (const file of filesToProcess) {
const duration = await getAudioDuration(file);
const currentTotal = referenceAudiosRef.current.reduce(
(sum, a) => sum + a.duration,
0,
);
if (currentTotal + duration > maxAudioTotalSec) {
toast.error(`Total audio duration cannot exceed ${maxAudioTotalSec}s`);
break;
}
const entry = makeEntry(file);
setUploadingAudio(entry);
const commit = (mediaToken: string) => {
const refAudio = asAudio({
id: randomId(),
url: entry.previewUrl,
file,
mediaToken,
duration,
});
setUploadingAudio(null);
setReferenceAudios?.([...referenceAudiosRef.current, refAudio]);
};
if (uploadAudio) {
await uploadAudio({
title: randomTitle("reference-audio"),
assetFile: file,
progressCallback: (newState) => {
if (newState.status === UploaderStates.success && newState.data) {
commit(newState.data);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
URL.revokeObjectURL(entry.previewUrl);
setUploadingAudio(null);
}
},
});
} else {
commit("");
}
}
};
const handleAudioFileUpload = async (
event: React.ChangeEvent<HTMLInputElement>,
) => {
const files = Array.from(event.target.files || []);
if (audioFileInputRef.current) audioFileInputRef.current.value = "";
if (files.length === 0) return;
await processAudioFiles(files);
};
// Combined picker for direct clicks on the reference card / circular "+":
// accepts every supported media kind and routes each file by MIME type.
const handleAnyFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
const files = Array.from(event.target.files || []);
if (anyFileInputRef.current) anyFileInputRef.current.value = "";
if (files.length === 0) return;
const images = files.filter((f) => f.type.startsWith("image/"));
const videos = files.filter((f) => f.type.startsWith("video/"));
const audios = files.filter((f) => f.type.startsWith("audio/"));
if (images.length > 0) processImageFiles(images, "start");
if (videos.length > 0 && setReferenceVideos) processVideoFiles(videos);
if (audios.length > 0 && setReferenceAudios) processAudioFiles(audios);
};
const anyUploadAccept = [
"image/*",
...(setReferenceVideos ? ["video/mp4", ".mp4"] : []),
...(setReferenceAudios ? ["audio/*"] : []),
].join(",");
const handleGalleryClose = () => {
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
};
const handleImageSelect = (id: string) => {
setSelectedGalleryImages((prev) => {
if (prev.includes(id)) return prev.filter((x) => x !== id);
const maxSelections =
galleryTarget === "video"
? Math.max(1, maxVideos - referenceVideos.length)
: galleryTarget === "audio"
? Math.max(1, maxAudios - referenceAudios.length)
: galleryTarget === "end"
? 1
: Math.max(1, maxImages);
if (prev.length >= maxSelections) {
return maxSelections === 1 ? [id] : prev;
}
return [...prev, id];
});
};
const handleGalleryImages = async (selectedItems: GalleryItem[]) => {
if (galleryTarget === "video") {
// Snapshot committed state at call time to avoid re-reading a stale
// ref after removes.
const baseVideos = [...referenceVideos];
const availableSlots = Math.max(0, maxVideos - baseVideos.length);
if (availableSlots <= 0) {
toast.error(`Max ${maxVideos} videos / ${maxVideoTotalSec}s total`, {
id: "video-ref-limit",
});
handleGalleryClose();
return;
}
const itemsToProcess = selectedItems
.slice(0, availableSlots)
.filter((item): item is GalleryItem & { fullImage: string } =>
Boolean(item.fullImage),
);
setIsProcessingGallery(true);
try {
// Parallelize duration probes — sequential metadata loads was the
// source of the perceived lag on the "Use selected" click.
const durations = await Promise.all(
itemsToProcess.map((item) => getVideoDurationFromSrc(item.fullImage)),
);
const newVideos: TVideo[] = [];
let currentTotal = baseVideos.reduce((sum, v) => sum + v.duration, 0);
let exceeded = false;
for (let i = 0; i < itemsToProcess.length; i++) {
const item = itemsToProcess[i]!;
const duration = durations[i]!;
if (currentTotal + duration > maxVideoTotalSec) {
exceeded = true;
break;
}
currentTotal += duration;
newVideos.push(
asVideo({
id: randomId(),
url: item.fullImage,
file: new File([], "library-video"),
mediaToken: item.id,
duration,
}),
);
}
if (exceeded) {
toast.error(
`Total video duration cannot exceed ${maxVideoTotalSec}s`,
{ id: "video-ref-limit" },
);
}
if (newVideos.length > 0) {
setReferenceVideos?.([...baseVideos, ...newVideos]);
}
} finally {
setIsProcessingGallery(false);
}
handleGalleryClose();
return;
}
if (galleryTarget === "audio") {
const baseAudios = [...referenceAudios];
const availableSlots = Math.max(0, maxAudios - baseAudios.length);
if (availableSlots <= 0) {
toast.error(`Max ${maxAudios} audio tracks / ${maxAudioTotalSec}s total`, {
id: "audio-ref-limit",
});
handleGalleryClose();
return;
}
const itemsToProcess = selectedItems
.slice(0, availableSlots)
.filter((item): item is GalleryItem & { fullImage: string } =>
Boolean(item.fullImage),
);
setIsProcessingGallery(true);
try {
// Use the duration the list endpoint already knows; probe the file's
// metadata only when it doesn't.
const durations = await Promise.all(
itemsToProcess.map((item) =>
item.durationMillis != null
? Promise.resolve(Math.round(item.durationMillis / 1000))
: getAudioDurationFromSrc(item.fullImage),
),
);
const newAudios: TAudio[] = [];
let currentTotal = baseAudios.reduce((sum, a) => sum + a.duration, 0);
let exceeded = false;
for (let i = 0; i < itemsToProcess.length; i++) {
const item = itemsToProcess[i]!;
const duration = durations[i]!;
if (currentTotal + duration > maxAudioTotalSec) {
exceeded = true;
break;
}
currentTotal += duration;
newAudios.push(
asAudio({
id: randomId(),
url: item.fullImage,
file: new File([], "library-audio"),
mediaToken: item.id,
duration,
}),
);
}
if (exceeded) {
toast.error(
`Total audio duration cannot exceed ${maxAudioTotalSec}s`,
{ id: "audio-ref-limit" },
);
}
if (newAudios.length > 0) {
setReferenceAudios?.([...baseAudios, ...newAudios]);
}
} finally {
setIsProcessingGallery(false);
}
handleGalleryClose();
return;
}
if (galleryTarget === "end") {
const item = selectedItems[0];
if (item && item.fullImage) {
setEndFrameImage?.(
asImage({
id: randomId(),
url: item.fullImage,
file: new File([], "library-image"),
mediaToken: item.id,
}),
);
}
handleGalleryClose();
return;
}
const availableSlots = Math.max(0, maxImages - referenceImages.length);
if (availableSlots <= 0) {
handleGalleryClose();
return;
}
const newRefs = [...referenceImages];
selectedItems.slice(0, availableSlots).forEach((item) => {
if (!item.fullImage) return;
newRefs.push(
asImage({
id: randomId(),
url: item.fullImage,
file: new File([], "library-image"),
mediaToken: item.id,
}),
);
});
setReferenceImages(newRefs);
handleGalleryClose();
};
const availableImageSlots = Math.max(
0,
maxImages - referenceImages.length - uploadingImages.length,
);
const fileInputs: ReactNode = (
<>
<input
type="file"
ref={fileInputRef}
className="hidden"
accept="image/*"
onChange={handleFileUpload}
multiple={maxImages > 1}
/>
<input
type="file"
ref={anyFileInputRef}
className="hidden"
accept={anyUploadAccept}
onChange={handleAnyFileUpload}
multiple
/>
{(uploadVideo || setReferenceVideos) && (
<input
type="file"
ref={videoFileInputRef}
className="hidden"
accept="video/mp4,.mp4"
onChange={handleVideoFileUpload}
multiple={maxVideos > 1}
/>
)}
{(uploadAudio || setReferenceAudios) && (
<input
type="file"
ref={audioFileInputRef}
className="hidden"
accept="audio/*"
onChange={handleAudioFileUpload}
multiple={maxAudios > 1}
/>
)}
</>
);
const galleryModal: ReactNode = ownGalleryModal ? (
<GalleryModal
key={
galleryTarget === "video"
? "video"
: galleryTarget === "audio"
? "audio"
: "image"
}
isOpen={!!isGalleryModalOpen}
onClose={handleGalleryClose}
mode="select"
selectedItemIds={selectedGalleryImages}
onSelectItem={handleImageSelect}
maxSelections={
galleryTarget === "end"
? 1
: galleryTarget === "video"
? Math.max(1, maxVideos - referenceVideos.length)
: galleryTarget === "audio"
? Math.max(1, maxAudios - referenceAudios.length)
: Math.max(1, availableImageSlots)
}
onUseSelected={handleGalleryImages}
onDownloadClicked={downloadFileFromUrl}
useSelectedLoading={isProcessingGallery}
forceFilter={
galleryTarget === "video"
? "video"
: galleryTarget === "audio"
? "audio"
: "image"
}
/>
) : null;
return {
uploadingImages,
uploadingEnd,
uploadingVideo,
uploadingAudio,
availableImageSlots,
openImageUpload,
openEndUpload,
openVideoUpload,
openAudioUpload,
openAnyUpload,
openGallery,
fileInputs,
galleryModal,
};
}
@@ -187,7 +187,7 @@ export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
referenceAudios: [],
generateWithSound: true,
duration: null,
inputMode: "keyframe",
inputMode: "reference",
generationCount: 1,
setPrompt: (prompt) => set({ prompt }),
setResolution: (resolution) => set({ resolution }),