Compare commits

...
Author SHA1 Message Date
bflatastic 5a58a60475 Add Audio to webapp home cards; align ToggleButton height to 34px
- Home: new Audio card after Video linking to /create-audio (pink accent, faMusic)
- ToggleButton: h-9 (36px) -> h-[34px]/w-[34px] to match GenerateButton and popover trigger pills in promptbox toolbars
2026-07-07 23:38:47 +07:00
bflatastic 55fab85170 Add Create Audio pages (Suno + Seed Audio) with waveform gallery playback to desktop and webapp 2026-07-07 20:50:08 +07:00
77 changed files with 4619 additions and 541 deletions
@@ -0,0 +1 @@
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Suno</title><path d="M16.5 0C20.642 0 24 5.373 24 12h-9c0 6.627-3.358 12-7.5 12C3.358 24 0 18.627 0 12h9c0-6.627 3.358-12 7.5-12z"></path></svg>

After

Width:  |  Height:  |  Size: 308 B

@@ -14,6 +14,7 @@ import Media from "../pages/media";
import { ToastContainer } from "../components/toast/toast";
import CreateImage from "../pages/create-image";
import CreateVideo from "../pages/create-video";
import CreateAudio from "../pages/create-audio";
import CreateVFX from "../pages/create-vfx";
import PageScene from "../pages/pagescene";
import PageDraw from "../pages/pagedraw";
@@ -153,6 +154,7 @@ export function App() {
<Route path="/" element={<Home />} />
<Route path="/create-image" element={<CreateImage />} />
<Route path="/create-video" element={<CreateVideo />} />
<Route path="/create-audio" element={<CreateAudio />} />
<Route path="/background-change" element={<CreateVFX />} />
<Route path="/moodboard" element={<MoodboardPage />} />
<Route path="/edit-3d" element={<PageScene />} />
@@ -1,7 +1,12 @@
import { useCallback, useState } from "react";
import { useNavigate } from "react-router-dom";
import type { IconDefinition } from "@fortawesome/fontawesome-svg-core";
import { faCube, faImage, faVideo } from "@fortawesome/pro-solid-svg-icons";
import {
faCube,
faImage,
faMusic,
faVideo,
} from "@fortawesome/pro-solid-svg-icons";
import { toast } from "../toast/toast";
import { downloadMediaFile } from "../../lib/download-media";
import {
@@ -23,6 +28,7 @@ import type { GalleryItem } from "./useGalleryData";
export interface GalleryItemActions {
isVideo: boolean;
is3D: boolean;
isAudio: boolean;
mediaIcon: IconDefinition;
mediaLabel: string;
modelDisplayName: string | null;
@@ -57,15 +63,23 @@ export function useGalleryItemActions(
const isVideo = item.mediaClass === "video";
const is3D = item.mediaClass === "dimensional";
const isAudio = item.mediaClass === "audio";
// Audio v1: no recreate / make-video — download + share only.
const recreateMediaClass: RecreateMediaClass | null = isVideo
? "video"
: is3D
: is3D || isAudio
? null
: "image";
const canMakeVideo = enableMakeVideo && !isVideo && !is3D;
const canMakeVideo = enableMakeVideo && !isVideo && !is3D && !isAudio;
const mediaIcon = isVideo ? faVideo : is3D ? faCube : faImage;
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : "Image";
const mediaIcon = isVideo
? faVideo
: is3D
? faCube
: isAudio
? faMusic
: faImage;
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : isAudio ? "Audio" : "Image";
const effectiveModelId = modelId ?? item.modelId;
const modelDisplayName = effectiveModelId
@@ -135,6 +149,7 @@ export function useGalleryItemActions(
return {
isVideo,
is3D,
isAudio,
mediaIcon,
mediaLabel,
modelDisplayName,
@@ -1,406 +1,4 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { JobsApi, JobStatus, MediaFilesApi } from "@storyteller/api";
import type { Job, Prompts } from "@storyteller/api";
import { getMediaThumbnail, THUMBNAIL_SIZES } from "@storyteller/common";
import {
getModelDisplayName,
getProviderDisplayName,
ALL_MODELS_LIST,
} from "@storyteller/model-list";
import type { GalleryItem } from "./useGalleryData";
import { getCachedPrompt, usePrompts } from "../../lib/prompts-cache";
// ── Types ──────────────────────────────────────────────────────────────────
// The job shapes moved to the shared generation-list lib; re-exported here so
// existing import sites keep working.
// Moved to the shared generation-list lib (used by both webapp and desktop).
// Re-exported so every webapp consumer shares the lib's module-level caches.
export { useGenerationJobs } from "@storyteller/ui-generation-list";
export type { InProgressJob, FailedJob } from "@storyteller/ui-generation-list";
import type { InProgressJob, FailedJob } from "@storyteller/ui-generation-list";
// ── Constants ──────────────────────────────────────────────────────────────
const IN_PROGRESS_STATUSES = new Set([JobStatus.PENDING, JobStatus.STARTED]);
const COMPLETED_STATUSES = new Set([JobStatus.COMPLETE_SUCCESS]);
const FAILED_STATUSES = new Set([
JobStatus.ATTEMPT_FAILED,
JobStatus.COMPLETE_FAILURE,
JobStatus.DEAD,
JobStatus.CANCELLED_BY_USER,
JobStatus.CANCELLED_BY_SYSTEM,
]);
const FAILURE_REASON_LABEL: Record<string, string> = {
rule_bans_user_image: "Image violates content policy",
rule_bans_user_image_with_faces: "Images with faces are not allowed",
rule_bans_user_text_prompt: "Text prompt violates content policy",
rule_bans_user_content: "Content violates content policy",
rule_bans_generated_video: "Generated video flagged by content policy",
rule_bans_generated_audio: "Generated audio flagged by content policy",
rule_bans_generated_content: "Generated content flagged by content policy",
generation_failed: "Generation failed",
unknown: "An unknown error occurred",
};
// Cache per-task durations
const taskDurationCache = new Map<string, number>();
// ── Helpers ────────────────────────────────────────────────────────────────
function getJobMediaType(job: Job): "image" | "video" | "other" {
const cat = job.request.inference_category?.toLowerCase() ?? "";
if (cat.includes("video")) return "video";
if (cat.includes("image")) return "image";
return "other";
}
function getModelLabel(job: Job, promptsMap?: Map<string, Prompts>): string {
const promptToken = job.request.maybe_prompt_token;
const cachedPrompt = promptToken
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
: undefined;
const modelType =
cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
const providerKey = cachedPrompt?.maybe_generation_provider ?? modelType;
const modelDisplay = modelType ? getModelDisplayName(modelType) : undefined;
const provider = providerKey
? getProviderDisplayName(providerKey.toLowerCase())
: undefined;
if (modelDisplay && provider) return `${modelDisplay} · ${provider}`;
return modelDisplay || provider || "Unknown model";
}
function getPrompt(job: Job, promptsMap?: Map<string, Prompts>): string {
const promptToken = job.request.maybe_prompt_token;
const cached = promptToken
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
: undefined;
return (
cached?.maybe_positive_prompt || job.request.maybe_raw_inference_text || ""
);
}
function getModelId(job: Job, promptsMap?: Map<string, Prompts>): string {
const promptToken = job.request.maybe_prompt_token;
const cachedPrompt = promptToken
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
: undefined;
return cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
}
function jobToInProgress(
job: Job,
promptsMap: Map<string, Prompts>,
): InProgressJob {
const now = Date.now();
const createdMs = new Date(job.created_at).getTime();
const modelType = job.request.maybe_model_type;
const isVideo =
job.request.inference_category?.toLowerCase().includes("video") ?? false;
let duration = taskDurationCache.get(job.job_token);
if (!duration) {
const model = modelType
? ALL_MODELS_LIST.find(
(m) => m.tauriId === modelType || m.id === modelType,
)
: undefined;
duration = model?.progressBarTime ?? (isVideo ? 900000 : 30000);
taskDurationCache.set(job.job_token, duration);
}
const elapsed = now - createdMs;
const progress = Math.min(95, (elapsed / duration) * 100);
const estimatedTimeLeftMs = Math.max(0, duration - elapsed);
return {
id: job.job_token,
prompt: getPrompt(job, promptsMap),
modelId: getModelId(job, promptsMap),
modelLabel: getModelLabel(job, promptsMap),
progress,
estimatedTimeLeftMs,
createdAt: job.created_at,
promptToken: job.request.maybe_prompt_token ?? undefined,
mediaClass: isVideo ? "video" : "image",
};
}
function jobToFailed(job: Job, promptsMap: Map<string, Prompts>): FailedJob {
const failureCategory =
job.status.maybe_failure_category_updated ||
job.status.maybe_failure_category;
const rawMessage =
job.status.maybe_failure_message ||
job.status.maybe_extra_status_description;
const failureReason = failureCategory
? FAILURE_REASON_LABEL[failureCategory] || rawMessage || undefined
: rawMessage || undefined;
const failureMessage =
rawMessage && failureCategory !== "unknown" ? rawMessage : undefined;
const promptToken = job.request.maybe_prompt_token ?? undefined;
const promptData = promptToken ? promptsMap.get(promptToken) : undefined;
const refImageUrl = pickFirstRefImageUrl(promptData);
const inferred = getJobMediaType(job);
const mediaClass: "image" | "video" =
inferred === "video" ? "video" : "image";
return {
id: job.job_token,
prompt: getPrompt(job, promptsMap),
modelId: getModelId(job, promptsMap),
modelLabel: getModelLabel(job, promptsMap),
failureReason,
failureMessage,
status: job.status.status,
createdAt: job.created_at,
promptToken,
refImageUrl,
mediaClass,
};
}
// Semantics that aren't still images — videos and audio clips can't be rendered
// as the faded backdrop behind the error state, so skip them when picking the
// reference to display.
const NON_IMAGE_REF_SEMANTICS = new Set(["vid_ref", "audioref"]);
function pickFirstRefImageUrl(
promptData: Prompts | undefined,
): string | undefined {
const refs = promptData?.maybe_context_images;
if (!refs?.length) return undefined;
for (const ref of refs) {
if (NON_IMAGE_REF_SEMANTICS.has(ref.semantic)) continue;
const url = ref.media_links?.cdn_url;
if (url) return url;
}
return undefined;
}
function jobToGalleryItem(
job: Job,
promptsMap?: Map<string, Prompts>,
): GalleryItem | null {
const result = job.maybe_result;
if (!result?.entity_token) return null;
const mediaType = getJobMediaType(job);
const mediaClass = mediaType === "video" ? "video" : "image";
const thumbnail = getMediaThumbnail(result.media_links, mediaClass, {
size: THUMBNAIL_SIZES.LARGE,
});
return {
id: result.entity_token,
label: getPrompt(job, promptsMap) || "Generation",
thumbnail,
fullImage: result.media_links?.cdn_url || null,
// Sort by job creation time (not completion time) so the completed card
// occupies the same Masonry slot the pending card held — no layout shift.
createdAt: job.created_at,
mediaClass,
modelId: job.request.maybe_model_type ?? undefined,
};
}
/** Expand a single GalleryItem into its batch siblings (if any). */
async function expandBatchItems(
item: GalleryItem,
mediaFilesApi: MediaFilesApi,
): Promise<GalleryItem[]> {
try {
const mediaResponse = await mediaFilesApi.GetMediaFileByToken({
mediaFileToken: item.id,
});
const batchToken = (mediaResponse.data as any)?.maybe_batch_token;
if (!batchToken) return [item];
const batchResponse = await mediaFilesApi.GetMediaFilesByBatchToken({
batchToken,
});
if (!batchResponse.success || !batchResponse.data?.length) return [item];
return batchResponse.data
.map((file: any): GalleryItem | null => {
const cdnUrl = file.media_links?.cdn_url;
if (!cdnUrl) return null;
const thumbnail = getMediaThumbnail(file.media_links, item.mediaClass, {
size: THUMBNAIL_SIZES.LARGE,
});
return {
id: file.token,
label: item.label,
thumbnail: thumbnail || cdnUrl,
fullImage: cdnUrl,
createdAt: item.createdAt,
mediaClass: item.mediaClass,
modelId: item.modelId,
batchImageToken: batchToken,
};
})
.filter((i): i is GalleryItem => i !== null);
} catch {
return [item];
}
}
// ── Hook ───────────────────────────────────────────────────────────────────
export function useGenerationJobs(options: {
mediaType: "image" | "video";
enabled?: boolean;
}) {
const { mediaType, enabled = true } = options;
const apiRef = useRef(new JobsApi());
const mediaApiRef = useRef(new MediaFilesApi());
const [inProgressJobs, setInProgressJobs] = useState<Job[]>([]);
const [failedJobsRaw, setFailedJobsRaw] = useState<Job[]>([]);
const [newlyCompleted, setNewlyCompleted] = useState<GalleryItem[]>([]);
const prevCompletedIdsRef = useRef<Set<string>>(new Set());
const initialLoadDoneRef = useRef(false);
const promptTokens = useMemo(() => {
const tokens: string[] = [];
for (const j of inProgressJobs) {
if (j.request.maybe_prompt_token)
tokens.push(j.request.maybe_prompt_token);
}
for (const j of failedJobsRaw) {
if (j.request.maybe_prompt_token)
tokens.push(j.request.maybe_prompt_token);
}
return tokens;
}, [inProgressJobs, failedJobsRaw]);
const promptsMap = usePrompts(promptTokens);
const inProgress = useMemo(
() =>
inProgressJobs
.slice()
.sort(
(a, b) =>
new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
)
.map((j) => jobToInProgress(j, promptsMap)),
[inProgressJobs, promptsMap],
);
const failed = useMemo(
() =>
failedJobsRaw
.slice()
.sort(
(a, b) =>
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
)
.map((j) => jobToFailed(j, promptsMap)),
[failedJobsRaw, promptsMap],
);
const load = useCallback(async () => {
try {
const response = await apiRef.current.ListRecentJobs();
if (!response.success || !response.data) return;
const jobs: Job[] = response.data;
// Filter by media type
const filtered = jobs.filter((j) => getJobMediaType(j) === mediaType);
const newInProgress = filtered.filter((j) =>
IN_PROGRESS_STATUSES.has(j.status.status),
);
const newFailed = filtered.filter((j) =>
FAILED_STATUSES.has(j.status.status),
);
// Completed
const completedJobs = filtered.filter((j) =>
COMPLETED_STATUSES.has(j.status.status),
);
const completedIdSet = new Set(completedJobs.map((j) => j.job_token));
// Detect newly completed (skip on first load to avoid flooding)
let expandedNewItems: GalleryItem[] = [];
if (initialLoadDoneRef.current) {
const newOnes = completedJobs.filter(
(j) => !prevCompletedIdsRef.current.has(j.job_token),
);
if (newOnes.length > 0) {
const items = newOnes
.map((j) => jobToGalleryItem(j))
.filter((item): item is GalleryItem => item !== null);
if (items.length > 0) {
// Await expansion so the pending card and its completed replacement
// commit in the same React render — no "remove then add" gap.
const expanded = await Promise.all(
items.map((item) => expandBatchItems(item, mediaApiRef.current)),
);
expandedNewItems = expanded.flat();
}
}
}
initialLoadDoneRef.current = true;
prevCompletedIdsRef.current = completedIdSet;
// Prune duration cache
const activeIds = new Set(newInProgress.map((j) => j.job_token));
for (const id of taskDurationCache.keys()) {
if (!activeIds.has(id)) taskDurationCache.delete(id);
}
if (expandedNewItems.length > 0) {
setNewlyCompleted((prev) => {
const existingIds = new Set(prev.map((i) => i.id));
const fresh = expandedNewItems.filter((i) => !existingIds.has(i.id));
return [...fresh, ...prev];
});
}
setInProgressJobs(newInProgress);
setFailedJobsRaw(newFailed);
} catch {
// ignore
}
}, [mediaType]);
// Poll every 5 seconds + listen for task-queue-update events.
// Skip entirely when disabled (e.g. user is logged out) — otherwise we'd
// hit an authenticated endpoint every 5s for nothing, which on mobile
// Safari shows up as periodic main-thread jank during the menu animation.
useEffect(() => {
if (!enabled) return;
load();
const intervalId = setInterval(load, 5000);
const handleTaskUpdate = () => load();
window.addEventListener("task-queue-update", handleTaskUpdate);
return () => {
clearInterval(intervalId);
window.removeEventListener("task-queue-update", handleTaskUpdate);
};
}, [load, enabled]);
const dismissFailed = useCallback(async (jobToken: string) => {
try {
await apiRef.current.DeleteJobByToken(jobToken);
setFailedJobsRaw((prev) => prev.filter((f) => f.job_token !== jobToken));
} catch {
// ignore
}
}, []);
return {
inProgress,
failed,
newlyCompleted,
dismissFailed,
refresh: load,
};
}
@@ -8,17 +8,20 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faChevronLeft,
faChevronRight,
faMusic,
faTrashCan,
} from "@fortawesome/pro-solid-svg-icons";
import { addCorsParam, PLACEHOLDER_IMAGES } from "@storyteller/common";
import { ActionReminderModal } from "@storyteller/ui-action-reminder-modal";
import { Viewer3D } from "@storyteller/ui-viewer-3d";
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
import useEmblaCarousel from "embla-carousel-react";
import type { EmblaOptionsType } from "embla-carousel";
import {
createPromptData,
EMPTY_PROMPT,
is3DModelUrl,
isAudioUrl,
isVideoUrl,
type PromptData,
} from "./shared";
@@ -275,6 +278,9 @@ export function Lightbox({
const is3D = selectedImageUrl
? is3DModelUrl(selectedImageUrl)
: propMediaClass === "dimensional";
const isAudio = selectedImageUrl
? isAudioUrl(selectedImageUrl) || propMediaClass === "audio"
: propMediaClass === "audio";
// Keyboard navigation
useEffect(() => {
@@ -312,9 +318,10 @@ export function Lightbox({
}
}, [selectedMediaToken, mediaFilesApi, onDeleted, onClose]);
// Audio v1: no recreate / make-video / edit-on-canvas.
const recreateMediaClass: "image" | "video" | null = isVideo
? "video"
: is3D
: is3D || isAudio
? null
: "image";
@@ -328,14 +335,14 @@ export function Lightbox({
);
}, [selectedMediaToken, recreateMediaClass, navigate, onClose]);
const canMakeVideo = !isVideo && !is3D;
const canMakeVideo = !isVideo && !is3D && !isAudio;
const handleMakeVideo = useCallback(() => {
if (!selectedMediaToken || !selectedImageUrl || !canMakeVideo) return;
onClose();
applyMakeVideoFromImage(selectedMediaToken, selectedImageUrl, navigate);
}, [selectedMediaToken, selectedImageUrl, canMakeVideo, navigate, onClose]);
const canEditOnCanvas = !isVideo && !is3D;
const canEditOnCanvas = !isVideo && !is3D && !isAudio;
const handleEditOnCanvas = useCallback(() => {
if (!selectedMediaToken || !selectedImageUrl || !canEditOnCanvas) return;
onClose();
@@ -391,6 +398,19 @@ export function Lightbox({
>
<source src={selectedImageUrl} type="video/mp4" />
</video>
) : isAudio ? (
<div className="flex h-full w-full items-center justify-center px-4 sm:px-8">
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-white/[0.04] px-2 py-8">
<FontAwesomeIcon
icon={faMusic}
className="mx-auto mb-5 block text-3xl text-white/20"
/>
<WaveformAudioPlayer
key={selectedImageUrl}
src={selectedImageUrl}
/>
</div>
</div>
) : (
<div className="flex h-full w-full flex-col justify-center">
<div
@@ -465,7 +485,7 @@ export function Lightbox({
</div>
)}
{!mediaLoaded && selectedImageUrl && !isVideo && !is3D && (
{!mediaLoaded && selectedImageUrl && !isVideo && !is3D && !isAudio && (
<div className="absolute inset-0 flex items-center justify-center bg-black">
<LoadingSpinner className="h-12 w-12 text-base-fg" />
</div>
@@ -12,6 +12,7 @@ export const COPY_FEEDBACK_DURATION = 1500;
const VIDEO_EXTENSIONS = [".mp4", ".webm", ".mov", ".avi", ".mkv", ".m4v"];
const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz"];
const AUDIO_EXTENSIONS = [".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac"];
export const isVideoUrl = (url: string): boolean =>
VIDEO_EXTENSIONS.some((ext) => url.toLowerCase().includes(ext));
@@ -19,6 +20,9 @@ export const isVideoUrl = (url: string): boolean =>
export const is3DModelUrl = (url: string): boolean =>
MODEL_3D_EXTENSIONS.some((ext) => url.toLowerCase().includes(ext));
export const isAudioUrl = (url: string): boolean =>
AUDIO_EXTENSIONS.some((ext) => url.toLowerCase().includes(ext));
export interface ContextImage {
media_links: {
cdn_url: string;
@@ -70,6 +70,10 @@ interface PromptBoxProps {
// Model selector (rendered above the toolbar, typically hidden on desktop via lg:hidden)
modelSelector?: ReactNode;
// Secondary prompt row rendered directly under the main textarea
// (e.g. the audio style/genre prompt).
secondaryPromptRow?: ReactNode;
// @-mention support (enables colored prompt overlay + autocomplete)
mentionItems?: MentionItem[];
@@ -106,6 +110,7 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
onClearAllRefs,
mediaReferenceRow,
modelSelector,
secondaryPromptRow,
mentionItems,
maxPromptLength,
},
@@ -355,7 +360,18 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
<div className="relative flex flex-col">
{isImageRowVisible && renderImagePromptRow()}
{mediaReferenceRow}
{/* When the media row is the topmost band (no image row above it —
e.g. the audio page), clip it to rounded top corners; mid-stack
(video's start-frame row above) it stays square. */}
{mediaReferenceRow && (
<div
className={twMerge(
!isImageRowVisible && "sm:overflow-hidden sm:rounded-t-2xl",
)}
>
{mediaReferenceRow}
</div>
)}
<div
className={twMerge(
@@ -543,6 +559,10 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
</div>
</div>
{secondaryPromptRow && (
<div className="mt-2">{secondaryPromptRow}</div>
)}
{/* Toolbar */}
<div className="mt-2 flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
<div className="flex flex-wrap items-center gap-1.5 sm:gap-2">
@@ -6,6 +6,7 @@ import {
faHouse,
faVideo,
faImage,
faMusic,
faCube,
faFilm,
faObjectGroup,
@@ -56,6 +57,7 @@ const PRIMARY_ITEMS: NavItem[] = [{ label: "Home", href: "/", icon: faHouse }];
const CREATE_ITEMS_STATIC: NavItem[] = [
{ label: "Image", href: "/create-image", icon: faImage },
{ label: "Video", href: "/create-video", icon: faVideo },
{ label: "Audio", href: "/create-audio", icon: faMusic },
// Edit Image is hidden from the sidebar for now.
{ label: "Edit Image", href: "/edit-image", icon: faPencil },
{ label: "Edit 3D", href: "/edit-3d", icon: faCube },
@@ -111,6 +111,14 @@ export function useStage3DCostEstimate(): void {
}, [credits, setEstimatedCreditsForPage]);
}
// ── Audio cost estimate hook ─────────────────────────────────────────────
// Shared with the desktop PromptBoxAudio via the omni-gen lib.
export {
useAudioCostEstimate,
type AudioCostParams,
} from "@storyteller/omni-gen";
// ── Video cost estimate hook ─────────────────────────────────────────────
export interface VideoCostParams {
@@ -4,6 +4,7 @@ import { toast } from "../components/toast/toast";
const EXT_BY_MEDIA_CLASS: Record<string, string> = {
image: "png",
video: "mp4",
audio: "mp3",
dimensional: "glb",
};
@@ -24,6 +24,22 @@ export function resolveModelOption(
return options[0];
}
// Number twin of resolveModelOption for numeric option lists (e.g. audio
// sample rates). Same semantics: preference when supported, else model
// default, else first option; passthrough when the model exposes no options.
export function resolveModelNumberOption(
preferred: number | null | undefined,
options: readonly number[] | null | undefined,
modelDefault: number | null | undefined,
): number | undefined {
if (!options || options.length === 0) {
return preferred ?? modelDefault ?? undefined;
}
if (preferred != null && options.includes(preferred)) return preferred;
if (modelDefault != null && options.includes(modelDefault)) return modelDefault;
return options[0];
}
// Resolve a preferred batch count against a model's options/limits. Clamps to
// [1, max], then snaps to the nearest supported option ≤ the preference (so a
// user who wanted 4 keeps the highest the model allows), falling back to the
@@ -0,0 +1,77 @@
import { create } from "zustand";
import { persist } from "zustand/middleware";
import type { RefAudio, RefImage } from "../../components/prompt-box";
export type AudioUiState = {
selectedModelId: string | null;
prompt: string;
stylePrompt: string;
// Suno toggles
isInstrumental: boolean;
keepLyrics: boolean;
isLoopable: boolean;
// Suno Sounds beat controls. bpm null = "Auto" (omitted from the request).
bpm: number | null;
musicalKey: string;
// Seed Audio output shaping. sampleRateHz is resolved against the current
// model's options at read time (sticky across model switches).
sampleRateHz: number | null;
speed: number;
volume: number;
pitch: number;
};
type CreateAudioState = {
ui: AudioUiState;
referenceAudios: RefAudio[];
referenceImages: RefImage[];
setUi: (patch: Partial<AudioUiState>) => void;
setReferenceAudios: (audios: RefAudio[]) => void;
setReferenceImages: (images: RefImage[]) => void;
reset: () => void;
};
const DEFAULT_UI: AudioUiState = {
selectedModelId: null,
prompt: "",
stylePrompt: "",
isInstrumental: false,
keepLyrics: false,
isLoopable: false,
bpm: null,
musicalKey: "auto",
sampleRateHz: null,
speed: 1,
volume: 1,
pitch: 0,
};
export const useCreateAudioStore = create<CreateAudioState>()(
persist(
(set) => ({
ui: { ...DEFAULT_UI },
referenceAudios: [],
referenceImages: [],
setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })),
setReferenceAudios: (audios) => set({ referenceAudios: audios }),
setReferenceImages: (images) => set({ referenceImages: images }),
reset: () => set({ ui: { ...DEFAULT_UI } }),
}),
{
name: "artcraft-audio-prompt",
// Persist prompt + settings so a full page reload keeps the user's
// draft. Reference audios/images are excluded: File handles aren't
// serializable and uploads can exceed the localStorage quota.
partialize: (state) => ({ ui: state.ui }),
merge: (persisted, current) => {
const persistedUi = (persisted as { ui?: Partial<AudioUiState> } | null)
?.ui;
return { ...current, ui: { ...current.ui, ...(persistedUi ?? {}) } };
},
},
),
);
@@ -0,0 +1,870 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import { FilterMediaClasses } from "@storyteller/api";
import type { OmniGenAudioModelDetails } from "@storyteller/api";
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { Button, ToggleButton } from "@storyteller/ui-button";
import { SliderV2 } from "@storyteller/ui-sliderv2";
import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal";
import {
AudioTuningPopover,
SoundsSettingsPopover,
StylePromptRow,
MUSICAL_KEYS,
formatSampleRateHz,
AUDIO_BPM_MIN,
AUDIO_BPM_MAX,
AUDIO_SPEED_MIN,
AUDIO_SPEED_MAX,
AUDIO_VOLUME_MIN,
AUDIO_VOLUME_MAX,
AUDIO_PITCH_MIN,
AUDIO_PITCH_MAX,
} from "@storyteller/ui-promptbox";
import {
PromptBox,
ImagePromptRow,
MediaReferenceRow,
MobilePromptForm,
MobileSelectField,
MobileFieldButton,
SettingsDrawer,
DrawerOptionList,
DrawerSection,
type RefImage,
} from "../../components/prompt-box";
import {
GenerationGallery,
useGalleryData,
useGenerationJobs,
useAuthCheck,
usePromptHeight,
useLightboxNav,
CreateMediaPageShell,
} from "../../components/generation-gallery";
import { Lightbox } from "../../components/lightbox/lightbox";
import { useCreateAudioStore } from "./create-audio-store";
import {
enqueueAudioGeneration,
AUDIO_MODELS_REQUIRING_AUDIO_REF,
} from "./generate-audio-api";
import type { AudioGenerationSettings } from "./generate-audio-api";
import { useAudioCostEstimate } from "../../lib/cost-estimate-api";
import { resolveModelNumberOption } from "../../lib/resolve-model-setting";
import {
useOmniGenAudioModels,
OMNI_GENERATE_OUTAGE_MESSAGE,
} from "@storyteller/omni-gen";
import {
getCreatorIconPathForModelId,
getModelDescription,
getModelInfo,
} from "@storyteller/model-list";
import { toast } from "../../components/toast/toast";
import { useSignupCta } from "../../components/signup-cta-modal";
import { useInsufficientCredits } from "../../components/insufficient-credits-modal";
import {
faMicrophoneLines,
faMicrophoneSlash,
faRepeat,
faSparkles,
} from "@fortawesome/pro-solid-svg-icons";
// ── Constants ────────────────────────────────────────────────────────────
const DEFAULT_MODEL_ID = "suno_music";
const AUDIO_FILTER = [FilterMediaClasses.AUDIO];
const AUDIO_REF_MAX_DURATION_SECONDS = 600;
const supports = (flag: boolean | null | undefined): boolean => flag === true;
// Store API model data alongside popover items via a lookup map
let _modelLookup = new Map<string, OmniGenAudioModelDetails>();
function buildModelPopoverItems(
models: OmniGenAudioModelDetails[],
selectedId: string,
): PopoverItem[] {
_modelLookup = new Map(models.map((m) => [m.model, m]));
return models.map((model) => ({
label: model.full_name || model.model,
selected: model.model === selectedId,
description: getModelDescription(model.model, model.extra_info_short),
info: getModelInfo(model.model, model.extra_info) || undefined,
icon: (
<img
src={getCreatorIconPathForModelId(model.model)}
alt={`${model.model} logo`}
className="h-4 w-4 icon-auto-contrast"
/>
),
action: model.model, // use action to carry the model id
}));
}
// ── Component ────────────────────────────────────────────────────────────
export default function CreateAudio() {
const { user, authChecked } = useAuthCheck();
const { loggedIn, openSignupCta } = useSignupCta();
const openInsufficientCredits = useInsufficientCredits();
const { promptBoxRef, promptHeight } = usePromptHeight();
// Fetch models from API
const { models: apiModels } = useOmniGenAudioModels();
// UI state
const ui = useCreateAudioStore((s) => s.ui);
const setUi = useCreateAudioStore((s) => s.setUi);
const selectedModel = useMemo(
(): OmniGenAudioModelDetails | undefined => {
if (!apiModels.length) return undefined;
if (ui.selectedModelId) {
return (
apiModels.find((m) => m.model === ui.selectedModelId) ??
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ??
apiModels[0]
);
}
return (
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0]
);
},
[apiModels, ui.selectedModelId],
);
const prompt = ui.prompt;
const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]);
const setStylePrompt = useCallback(
(v: string) => setUi({ stylePrompt: v }),
[setUi],
);
// Capability gating. Flags are serde-skipped when absent — only `true`
// counts as supported.
const styleSupported = supports(selectedModel?.style_prompt_supported);
const hasInstrumental = supports(
selectedModel?.instrumental_toggle_supported,
);
const hasKeepLyrics = supports(selectedModel?.keep_lyrics_supported);
const hasLoopable = supports(selectedModel?.loopable_toggle_supported);
const hasBpm = supports(selectedModel?.bpm_supported);
const hasMusicalKey = supports(selectedModel?.musical_key_supported);
const sampleRateOptions = selectedModel?.sample_rate_hz_options ?? null;
const hasSpeed = supports(selectedModel?.speed_supported);
const hasVolume = supports(selectedModel?.volume_supported);
const hasPitch = supports(selectedModel?.pitch_supported);
const hasTuning =
!!sampleRateOptions?.length || hasSpeed || hasVolume || hasPitch;
const audioRefsSupported = supports(
selectedModel?.audio_references_supported,
);
const maxAudioRefs = selectedModel?.audio_references_max ?? 1;
const imageRefsSupported = supports(
selectedModel?.image_references_supported,
);
const maxImageRefs = selectedModel?.image_references_max ?? 1;
const requiresAudioRef = AUDIO_MODELS_REQUIRING_AUDIO_REF.has(
selectedModel?.model ?? "",
);
// Seed Audio output shaping — sticky across model switches; numeric sliders
// clamp at read time so persisted values always stay in the valid range.
const sampleRateHz =
resolveModelNumberOption(
ui.sampleRateHz,
sampleRateOptions,
selectedModel?.sample_rate_hz_default,
) ?? null;
const speed = clamp(ui.speed, AUDIO_SPEED_MIN, AUDIO_SPEED_MAX);
const volume = clamp(ui.volume, AUDIO_VOLUME_MIN, AUDIO_VOLUME_MAX);
const pitch = clamp(ui.pitch, AUDIO_PITCH_MIN, AUDIO_PITCH_MAX);
const [isGenerating, setIsGenerating] = useState(false);
const referenceAudios = useCreateAudioStore((s) => s.referenceAudios);
const setReferenceAudios = useCreateAudioStore((s) => s.setReferenceAudios);
const referenceImages = useCreateAudioStore((s) => s.referenceImages);
const setReferenceImages = useCreateAudioStore((s) => s.setReferenceImages);
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
const [isBeatDrawerOpen, setIsBeatDrawerOpen] = useState(false);
const [isTuningDrawerOpen, setIsTuningDrawerOpen] = useState(false);
useEffect(() => {
if (isImagePickerOpen) setPickerSelectedIds([]);
}, [isImagePickerOpen]);
const handlePickerSelect = useCallback(
(id: string) => {
setPickerSelectedIds((prev) => {
if (prev.includes(id)) return prev.filter((x) => x !== id);
if (prev.length >= maxImageRefs) {
return maxImageRefs === 1 ? [id] : prev;
}
return [...prev, id];
});
},
[maxImageRefs],
);
// Jobs + gallery
const jobs = useGenerationJobs({ mediaType: "audio", enabled: !!user });
const gallery = useGalleryData({
username: user?.username ?? null,
filterMediaClasses: AUDIO_FILTER,
excludeUploads: true,
});
const newlyCompletedTokens = useMemo(
() => new Set(jobs.newlyCompleted.map((i) => i.id)),
[jobs.newlyCompleted],
);
// Lightbox
const flatItems = useMemo(() => {
const filtered = gallery.items.filter(
(i) => !newlyCompletedTokens.has(i.id),
);
return [...jobs.newlyCompleted, ...filtered];
}, [jobs.newlyCompleted, gallery.items, newlyCompletedTokens]);
const lightbox = useLightboxNav(flatItems);
const estimatedCredits = useAudioCostEstimate({
model: selectedModel?.model ?? "",
audioReferenceCount: referenceAudios.length,
hasImageReference: referenceImages.length > 0,
sampleRateHz: sampleRateOptions?.length ? sampleRateHz : undefined,
});
const modelItems = useMemo(
() => buildModelPopoverItems(apiModels, selectedModel?.model ?? ""),
[apiModels, selectedModel?.model],
);
const hasContent =
jobs.inProgress.length > 0 ||
jobs.failed.length > 0 ||
jobs.newlyCompleted.length > 0 ||
gallery.items.length > 0 ||
gallery.isInitialLoading;
const missingRequiredAudioRef =
requiresAudioRef && referenceAudios.length !== 1;
const canGenerate =
!!prompt.trim() && !isGenerating && !missingRequiredAudioRef;
// ── Handlers ───────────────────────────────────────────────────────────
const handleModelChange = useCallback(
(item: PopoverItem) => {
const model = item.action ? _modelLookup.get(item.action) : undefined;
if (!model) return;
// Only switch the model — style/toggles/tuning are preserved and
// resolved against the new model at read time, so the user's choices
// survive model switches instead of resetting to defaults.
setUi({ selectedModelId: model.model });
},
[setUi],
);
// Seed Audio can't combine audio and image references — adding one kind
// clears the other so the request is always valid.
const handleReferenceAudiosChange = useCallback(
(audios: typeof referenceAudios) => {
if (audios.length > 0 && referenceImages.length > 0) {
setReferenceImages([]);
toast.error("Removed image reference — it can't be combined with audio");
}
setReferenceAudios(audios);
},
[referenceImages, setReferenceAudios, setReferenceImages],
);
const handleReferenceImagesChange = useCallback(
(images: RefImage[]) => {
if (images.length > 0 && referenceAudios.length > 0) {
setReferenceAudios([]);
toast.error("Removed audio reference — it can't be combined with an image");
}
setReferenceImages(images);
},
[referenceAudios, setReferenceAudios, setReferenceImages],
);
const handleLibraryImageSelect = useCallback(
(items: GalleryItem[]) => {
const newImages: RefImage[] = items.slice(0, maxImageRefs).map((item) => ({
id: Math.random().toString(36).substring(7),
url: item.thumbnail || item.fullImage || "",
fullUrl: item.fullImage || undefined,
file: new File([], "library-image"),
mediaToken: item.id,
}));
handleReferenceImagesChange(newImages);
setIsImagePickerOpen(false);
},
[maxImageRefs, handleReferenceImagesChange],
);
const handleGenerate = useCallback(async () => {
if (!loggedIn) {
openSignupCta();
return;
}
if (!prompt.trim() || isGenerating || !selectedModel) return;
if (missingRequiredAudioRef) {
toast.error(
`${selectedModel.full_name ?? "This model"} needs an audio track to work from — add one first`,
);
return;
}
setIsGenerating(true);
try {
const settings: AudioGenerationSettings = {
prompt,
stylePrompt: ui.stylePrompt,
audioMediaTokens: referenceAudios
.map((a) => a.mediaToken)
.filter((t): t is string => typeof t === "string" && t.length > 0),
imageMediaTokens: referenceImages
.map((img) => img.mediaToken)
.filter((t): t is string => typeof t === "string" && t.length > 0),
isInstrumental: ui.isInstrumental,
keepLyrics: ui.keepLyrics,
isLoopable: ui.isLoopable,
bpm: ui.bpm,
musicalKey: ui.musicalKey,
sampleRateHz,
speed,
volume,
pitch,
};
const result = await enqueueAudioGeneration(selectedModel, settings);
if (!result.success) {
// 402 Payment Required: the user is out of credits. Surface the
// upgrade modal instead of an error toast.
if (result.errorCode === 402) {
openInsufficientCredits();
} else if (result.errorCode != null && result.errorCode >= 500) {
toast.error(OMNI_GENERATE_OUTAGE_MESSAGE);
} else {
toast.error(result.error ?? "Failed to start generation");
}
setIsGenerating(false);
return;
}
// The jobs feed (5s poll + this nudge) renders the pending cards — one
// per job token (Suno can return several clips per request).
window.dispatchEvent(new Event("credits-change"));
window.dispatchEvent(new Event("task-queue-update"));
} catch {
toast.error("Network error - please try again");
} finally {
setIsGenerating(false);
}
}, [
loggedIn,
openSignupCta,
openInsufficientCredits,
prompt,
isGenerating,
selectedModel,
missingRequiredAudioRef,
ui.stylePrompt,
ui.isInstrumental,
ui.keepLyrics,
ui.isLoopable,
ui.bpm,
ui.musicalKey,
sampleRateHz,
speed,
volume,
pitch,
referenceAudios,
referenceImages,
]);
// ── Toolbar pieces (shared between desktop promptbox + fullscreen) ────
const toggleButtons = (
<>
{hasInstrumental && (
<Tooltip
content={ui.isInstrumental ? "Instrumental: ON" : "Instrumental: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={ui.isInstrumental}
icon={faMicrophoneSlash}
activeIcon={faMicrophoneSlash}
label="Instrumental"
onClick={() => setUi({ isInstrumental: !ui.isInstrumental })}
/>
</Tooltip>
)}
{hasKeepLyrics && (
<Tooltip
content={ui.keepLyrics ? "Keep lyrics: ON" : "Keep lyrics: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={ui.keepLyrics}
icon={faMicrophoneLines}
activeIcon={faMicrophoneLines}
label="Keep lyrics"
onClick={() => setUi({ keepLyrics: !ui.keepLyrics })}
/>
</Tooltip>
)}
{hasLoopable && (
<Tooltip
content={ui.isLoopable ? "Loop: ON" : "Loop: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={ui.isLoopable}
icon={faRepeat}
activeIcon={faRepeat}
label="Loop"
onClick={() => setUi({ isLoopable: !ui.isLoopable })}
/>
</Tooltip>
)}
</>
);
const leftToolbar = (
<>
{toggleButtons}
{(hasBpm || hasMusicalKey) && (
<SoundsSettingsPopover
showBpm={hasBpm}
bpm={ui.bpm}
onBpmChange={(bpm) => setUi({ bpm })}
showMusicalKey={hasMusicalKey}
musicalKey={ui.musicalKey}
onMusicalKeyChange={(musicalKey) => setUi({ musicalKey })}
/>
)}
{hasTuning && (
<AudioTuningPopover
sampleRateOptions={sampleRateOptions}
sampleRateHz={sampleRateHz}
onSampleRateChange={(hz) => setUi({ sampleRateHz: hz })}
showSpeed={hasSpeed}
speed={speed}
onSpeedChange={(v) => setUi({ speed: v })}
showVolume={hasVolume}
volume={volume}
onVolumeChange={(v) => setUi({ volume: v })}
showPitch={hasPitch}
pitch={pitch}
onPitchChange={(v) => setUi({ pitch: v })}
/>
)}
</>
);
const audioReferenceRow = audioRefsSupported ? (
<MediaReferenceRow
videoSupported={false}
audioSupported
referenceVideos={[]}
onReferenceVideosChange={() => {}}
maxVideoCount={0}
maxVideoRefDuration={0}
referenceAudios={referenceAudios}
onReferenceAudiosChange={handleReferenceAudiosChange}
maxAudioCount={maxAudioRefs}
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
/>
) : undefined;
// ── Mobile form ────────────────────────────────────────────────────────
const beatSummary = [
hasBpm ? (ui.bpm === null ? "Auto BPM" : `${ui.bpm} BPM`) : null,
hasMusicalKey
? (MUSICAL_KEYS.find((k) => k.value === ui.musicalKey)?.label ?? "Auto")
: null,
]
.filter(Boolean)
.join(" · ");
const tuningSummary = [
sampleRateOptions?.length && sampleRateHz
? formatSampleRateHz(sampleRateHz)
: null,
hasSpeed ? `${speed.toFixed(2)}×` : null,
hasPitch && pitch !== 0 ? `${pitch > 0 ? "+" : ""}${pitch} st` : null,
]
.filter(Boolean)
.join(" · ");
const mobileForm = (
<MobilePromptForm
prompt={prompt}
onPromptChange={setPrompt}
onSubmit={handleGenerate}
isSubmitting={isGenerating}
credits={estimatedCredits}
placeholder="Describe the music or sound you want..."
autoAdvance={loggedIn && canGenerate}
modelField={
<MobileSelectField
label="Model"
title="Select Model"
items={modelItems}
onSelect={handleModelChange}
/>
}
frames={
imageRefsSupported ? (
<ImagePromptRow
maxImagePromptCount={maxImageRefs}
referenceImages={referenceImages}
setReferenceImages={handleReferenceImagesChange}
onPickFromLibrary={() => setIsImagePickerOpen(true)}
/>
) : undefined
}
mediaRefs={audioReferenceRow}
settingsFields={
<>
{styleSupported && (
<div className="rounded-xl border border-ui-panel-border bg-ui-controls px-3 py-1.5">
<StylePromptRow
value={ui.stylePrompt}
onChange={setStylePrompt}
className="border-t-0 pt-0"
/>
</div>
)}
{(hasInstrumental || hasKeepLyrics || hasLoopable) && (
<div className="flex flex-wrap items-center gap-2">
{toggleButtons}
</div>
)}
{(hasBpm || hasMusicalKey) && (
<>
<MobileFieldButton
label="Beat & Key"
value={beatSummary || "Auto"}
onClick={() => setIsBeatDrawerOpen(true)}
/>
<SettingsDrawer
open={isBeatDrawerOpen}
onOpenChange={setIsBeatDrawerOpen}
title="Beat & Key"
>
{hasBpm && (
<DrawerSection label="BPM">
<MobileSliderRow
valueLabel={ui.bpm === null ? "Auto" : `${ui.bpm}`}
min={AUDIO_BPM_MIN}
max={AUDIO_BPM_MAX}
step={1}
value={ui.bpm ?? 120}
onChange={(bpm) => setUi({ bpm })}
onReset={
ui.bpm !== null ? () => setUi({ bpm: null }) : undefined
}
/>
</DrawerSection>
)}
{hasMusicalKey && (
<DrawerSection label="Musical key">
<DrawerOptionList
items={MUSICAL_KEYS.map((key) => ({
label: key.label,
selected: key.value === ui.musicalKey,
action: key.value,
}))}
onSelect={(item) => {
if (item.action) setUi({ musicalKey: item.action });
}}
/>
</DrawerSection>
)}
</SettingsDrawer>
</>
)}
{hasTuning && (
<>
<MobileFieldButton
label="Tuning"
value={tuningSummary || "Default"}
onClick={() => setIsTuningDrawerOpen(true)}
/>
<SettingsDrawer
open={isTuningDrawerOpen}
onOpenChange={setIsTuningDrawerOpen}
title="Tuning"
>
{!!sampleRateOptions?.length && (
<DrawerSection label="Sample rate">
<DrawerOptionList
items={sampleRateOptions.map((hz) => ({
label: formatSampleRateHz(hz),
selected: hz === sampleRateHz,
action: String(hz),
}))}
onSelect={(item) => {
if (item.action)
setUi({ sampleRateHz: Number(item.action) });
}}
/>
</DrawerSection>
)}
{hasSpeed && (
<DrawerSection label="Speed">
<MobileSliderRow
valueLabel={`${speed.toFixed(2)}×`}
min={AUDIO_SPEED_MIN}
max={AUDIO_SPEED_MAX}
step={0.05}
value={speed}
onChange={(v) => setUi({ speed: v })}
/>
</DrawerSection>
)}
{hasVolume && (
<DrawerSection label="Volume">
<MobileSliderRow
valueLabel={`${volume.toFixed(2)}×`}
min={AUDIO_VOLUME_MIN}
max={AUDIO_VOLUME_MAX}
step={0.05}
value={volume}
onChange={(v) => setUi({ volume: v })}
/>
</DrawerSection>
)}
{hasPitch && (
<DrawerSection label="Pitch">
<MobileSliderRow
valueLabel={`${pitch > 0 ? "+" : ""}${pitch} st`}
min={AUDIO_PITCH_MIN}
max={AUDIO_PITCH_MAX}
step={1}
value={pitch}
onChange={(v) => setUi({ pitch: v })}
/>
</DrawerSection>
)}
</SettingsDrawer>
</>
)}
</>
}
/>
);
// ── Render ─────────────────────────────────────────────────────────────
return (
<CreateMediaPageShell
title="Create Audio - ArtCraft"
description="Generate music and sound effects with ArtCraft"
authChecked={authChecked}
hasContent={hasContent}
emptyStateTitle="Create Audio"
emptyStateSubtitle="Describe a song, a sound, or a sample. Hear it in minutes."
emptyStateCta={
loggedIn ? undefined : (
<Button
variant="primary"
onClick={openSignupCta}
icon={faSparkles}
className="h-12 px-6 text-base font-semibold rounded-full"
>
Sign up to create
</Button>
)
}
bottomOffset={promptHeight + 24}
modelItems={modelItems}
onModelChange={handleModelChange}
promptForm={mobileForm}
gridContent={
<GenerationGallery
inProgressJobs={jobs.inProgress}
failedJobs={jobs.failed}
onDismissFailed={jobs.dismissFailed}
newlyCompletedItems={jobs.newlyCompleted}
galleryItems={gallery.items}
newlyCompletedTokens={newlyCompletedTokens}
hasMore={gallery.hasMore}
isLoading={gallery.isLoading}
isInitialLoading={gallery.isInitialLoading}
onLoadMore={gallery.loadMore}
onGalleryItemClick={lightbox.handleGalleryItemClick}
/>
}
promptBox={
<div
ref={promptBoxRef}
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
style={{
animationDelay: "150ms",
left: "var(--ac-sidebar-offset, 0px)",
}}
>
<PromptBox
prompt={prompt}
onPromptChange={setPrompt}
onSubmit={handleGenerate}
isSubmitting={isGenerating}
disabled={!canGenerate}
credits={estimatedCredits}
placeholder="Describe the music or sound you want..."
supportsImagePrompts={imageRefsSupported}
maxImagePromptCount={maxImageRefs}
referenceImages={referenceImages}
onReferenceImagesChange={handleReferenceImagesChange}
onPickFromLibrary={() => setIsImagePickerOpen(true)}
mediaReferenceRow={audioReferenceRow}
secondaryPromptRow={
styleSupported ? (
<StylePromptRow
value={ui.stylePrompt}
onChange={setStylePrompt}
/>
) : undefined
}
modelSelector={
<Tooltip
content="Model"
position="top"
className="z-50"
closeOnClick
>
<PopoverMenu
items={modelItems}
onSelect={handleModelChange}
mode="toggle"
panelTitle="Select Model"
panelClassName="w-[360px]"
richList
triggerIcon={
<img
src={getCreatorIconPathForModelId(
selectedModel?.model ?? "",
)}
alt=""
className="h-4 w-4 icon-auto-contrast"
/>
}
/>
</Tooltip>
}
leftToolbar={leftToolbar}
rightToolbar={
missingRequiredAudioRef ? (
<span className="flex items-center gap-1.5 text-xs font-medium text-red-500 animate-pulse">
Audio track required
</span>
) : undefined
}
/>
</div>
}
modals={
<>
<GalleryModal
mode="select"
isOpen={isImagePickerOpen}
onClose={() => setIsImagePickerOpen(false)}
selectedItemIds={pickerSelectedIds}
onSelectItem={handlePickerSelect}
maxSelections={maxImageRefs}
onUseSelected={handleLibraryImageSelect}
forceFilter="image"
hideFilter
/>
<Lightbox
isOpen={lightbox.lightboxOpen}
onClose={lightbox.closeLightbox}
mediaToken={lightbox.lightboxItem?.id}
cdnUrl={lightbox.lightboxItem?.fullImage}
mediaClass={lightbox.lightboxItem?.mediaClass}
batchImageToken={lightbox.lightboxItem?.batchImageToken}
showBatchCarousel={false}
onNavigatePrev={lightbox.navigatePrev}
onNavigateNext={lightbox.navigateNext}
onDeleted={gallery.removeItem}
/>
</>
}
/>
);
}
// ── Helpers ──────────────────────────────────────────────────────────────
function clamp(value: number, min: number, max: number): number {
return Math.min(Math.max(value, min), max);
}
interface MobileSliderRowProps {
valueLabel: string;
min: number;
max: number;
step: number;
value: number;
onChange: (value: number) => void;
onReset?: () => void;
}
function MobileSliderRow({
valueLabel,
min,
max,
step,
value,
onChange,
onReset,
}: MobileSliderRowProps) {
return (
<div className="px-1 py-1.5">
<div className="mb-1.5 flex items-center justify-between">
<span className="text-sm font-medium tabular-nums text-base-fg">
{valueLabel}
</span>
{onReset && (
<button
type="button"
onClick={onReset}
className="rounded bg-white/5 px-2 py-0.5 text-xs font-medium text-base-fg/60 transition-colors hover:bg-white/10 hover:text-base-fg"
>
Auto
</button>
)}
</div>
<SliderV2
min={min}
max={max}
step={step}
value={value}
onChange={onChange}
variant="filled"
/>
</div>
);
}
@@ -0,0 +1,9 @@
// The audio request builder + enqueue live in the shared omni-gen lib (used
// by both this page and the desktop PromptBoxAudio). Re-exported so this
// page keeps the same local-module structure as create-image / create-video.
export {
buildAudioRequest,
enqueueAudioGeneration,
AUDIO_MODELS_REQUIRING_AUDIO_REF,
} from "@storyteller/omni-gen";
export type { AudioGenerationSettings } from "@storyteller/omni-gen";
@@ -0,0 +1 @@
export { default } from "./create-audio";
@@ -4,6 +4,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faImage,
faVideo,
faMusic,
faWandMagicSparkles,
faCube,
faFilm,
@@ -47,6 +48,15 @@ const APPS: AppCard[] = [
iconBg: "bg-purple-500/25 border-purple-400/30",
iconColor: "text-purple-300",
},
{
label: "Audio",
description: "Generate songs and sound effects from prompts.",
href: "/create-audio",
icon: faMusic,
accent: "from-pink-500/20 to-pink-500/0",
iconBg: "bg-pink-500/25 border-pink-400/30",
iconColor: "text-pink-300",
},
{
label: "Edit 3D",
description: "Compose 3D scenes and render with AI cameras.",
@@ -34,6 +34,7 @@ import {
faBorderAll,
faCube,
faImage,
faMusic,
faVideo,
faPencil,
faTrashCan,
@@ -59,12 +60,14 @@ const FILTERS = [
{ id: "all", label: "All", icon: faBorderAll, route: "/library" },
{ id: "image", label: "Images", icon: faImage, route: "/library/images" },
{ id: "video", label: "Videos", icon: faVideo, route: "/library/videos" },
{ id: "audio", label: "Audio", icon: faMusic, route: "/library/audio" },
{ id: "meshes", label: "Meshes", icon: faCube, route: "/library/meshes" },
];
const ROUTE_TO_FILTER: Record<string, string> = {
images: "image",
videos: "video",
audio: "audio",
meshes: "meshes",
};
@@ -76,12 +79,15 @@ const getFilterMediaClass = (
return [FilterMediaClasses.IMAGE];
case "video":
return [FilterMediaClasses.VIDEO];
case "audio":
return [FilterMediaClasses.AUDIO];
case "meshes":
return [FilterMediaClasses.DIMENSIONAL];
default:
return [
FilterMediaClasses.IMAGE,
FilterMediaClasses.VIDEO,
FilterMediaClasses.AUDIO,
FilterMediaClasses.DIMENSIONAL,
];
}
@@ -1547,7 +1553,9 @@ function BulkThumb({ item }: { item: GalleryItem }) {
? faVideo
: item.mediaClass === "dimensional"
? faCube
: faImage;
: item.mediaClass === "audio"
? faMusic
: faImage;
const showImage = !!item.thumbnail && !failed;
return (
<div className="-ml-2 h-8 w-8 flex-shrink-0 overflow-hidden rounded border-2 border-ui-panel bg-black/30 first:ml-0">
@@ -79,6 +79,9 @@
{
"path": "../../libs/components/switch/tsconfig.lib.json"
},
{
"path": "../../libs/components/audio-player/tsconfig.lib.json"
},
{
"path": "../../libs/components/viewer-3d/tsconfig.lib.json"
},
@@ -65,6 +65,9 @@
{
"path": "../../libs/components/generation-list"
},
{
"path": "../../libs/components/audio-player"
},
{
"path": "../../libs/components/tab-selector"
},
@@ -0,0 +1 @@
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Suno</title><path d="M16.5 0C20.642 0 24 5.373 24 12h-9c0 6.627-3.358 12-7.5 12C3.358 24 0 18.627 0 12h9c0-6.627 3.358-12 7.5-12z"></path></svg>

After

Width:  |  Height:  |  Size: 308 B

@@ -0,0 +1 @@
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Suno</title><path d="M4 9.25c.69 0 1.25.56 1.25 1.25v3a1.25 1.25 0 1 1-2.5 0v-3c0-.69.56-1.25 1.25-1.25zM8 5.5c.69 0 1.25.56 1.25 1.25v10.5a1.25 1.25 0 1 1-2.5 0V6.75c0-.69.56-1.25 1.25-1.25zM12 2.75c.69 0 1.25.56 1.25 1.25v16a1.25 1.25 0 1 1-2.5 0V4c0-.69.56-1.25 1.25-1.25zM16 6.5c.69 0 1.25.56 1.25 1.25v8.5a1.25 1.25 0 1 1-2.5 0v-8.5c0-.69.56-1.25 1.25-1.25zM20 9.75c.69 0 1.25.56 1.25 1.25v2a1.25 1.25 0 1 1-2.5 0v-2c0-.69.56-1.25 1.25-1.25z"></path></svg>

After

Width:  |  Height:  |  Size: 626 B

@@ -116,13 +116,15 @@ function ItemActions({
const isVideo = item.mediaClass === "video";
const is3D = item.mediaClass === "dimensional";
const isAudio = item.mediaClass === "audio";
// Audio v1: no recreate / make-video — share + download only.
const recreateMediaClass: "image" | "video" | null = isVideo
? "video"
: is3D
: is3D || isAudio
? null
: "image";
const canRecreate = !!promptToken && !!recreateMediaClass;
const canMakeVideo = !!enableMakeVideo && !isVideo && !is3D;
const canMakeVideo = !!enableMakeVideo && !isVideo && !is3D && !isAudio;
const buttonClass =
variant === "card"
@@ -357,6 +357,8 @@ export const TopBar = ({ pageName }: Props) => {
return "Create Image";
case "VIDEO":
return "Create Video";
case "AUDIO":
return "Create Audio";
case "EDIT":
return "Edit Image";
case "VIDEO_FRAME_EXTRACTOR":
@@ -499,7 +501,8 @@ export const TopBar = ({ pageName }: Props) => {
<div className="flex justify-end gap-2" data-tauri-drag-region>
<div className="no-drag flex items-center gap-1.5">
{(tabStore.activeTabId === "IMAGE" ||
tabStore.activeTabId === "VIDEO") && <GalleryViewToggle />}
tabStore.activeTabId === "VIDEO" ||
tabStore.activeTabId === "AUDIO") && <GalleryViewToggle />}
<PopoverMenu
position="bottom"
align="center"
@@ -3,6 +3,7 @@ import {
faCube,
faFilm,
faImage,
faMusic,
faDroplet,
faPhotoFilm,
faGlobe,
@@ -23,6 +24,7 @@ import { useTabStore, TabId } from "~/pages/Stores/TabState";
export type AppId =
| "IMAGE"
| "VIDEO"
| "AUDIO"
| "EDIT"
| "2D"
| "3D"
@@ -58,6 +60,11 @@ export const APP_DESCRIPTORS: AppDescriptor[] = [
label: "Create Video",
icon: faFilm,
},
{
id: "AUDIO",
label: "Create Audio",
icon: faMusic,
},
{
id: "2D",
label: "Image Editor",
@@ -113,6 +120,11 @@ const APP_CARD_PALETTES: Record<string, AppCardPalette> = {
iconBg: "bg-amber-500/20 border-amber-400/30",
iconColor: "text-amber-300",
},
"create-audio": {
accent: "from-pink-500/20 to-pink-500/0",
iconBg: "bg-pink-500/20 border-pink-400/30",
iconColor: "text-pink-300",
},
"image-to-3d-object": {
accent: "from-emerald-500/20 to-emerald-500/0",
iconBg: "bg-emerald-500/20 border-emerald-400/30",
@@ -208,6 +220,15 @@ export const ALL_APPS: FullAppItem[] = [
action: "VIDEO",
color: "bg-amber-500/40",
},
{
id: "create-audio",
label: "Create Audio",
description: "Generate music and sound effects",
icon: faMusic,
category: "generate",
action: "AUDIO",
color: "bg-pink-500/40",
},
{
id: "image-to-3d-object",
label: "Image to 3D Object",
@@ -398,6 +419,7 @@ export const goToApp = (action?: string) => {
[
"IMAGE",
"VIDEO",
"AUDIO",
"2D",
"3D",
"VIDEO_FRAME_EXTRACTOR",
@@ -60,6 +60,7 @@ import { AppsIndexPage } from "./PageApps/AppsIndexPage";
import PageDraw from "./PageDraw/PageDraw";
import TextToImage from "./PageImage/TextToImage";
import ImageToVideo from "./PageVideo/ImageToVideo";
import CreateAudio from "./PageAudio/CreateAudio";
import { VideoFrameExtractor } from "./PageVideoFrameExtractor";
import { VideoWatermarkRemover } from "./PageVideoWatermarkRemover";
import { ImageWatermarkRemover } from "./PageImageWatermarkRemover";
@@ -247,6 +248,12 @@ const TabBody = ({ sceneToken }: { sceneToken?: string }) => {
<ImageToVideo />
</div>
);
case "AUDIO":
return (
<div>
<CreateAudio />
</div>
);
case "VIDEO_FRAME_EXTRACTOR":
return (
<div>
@@ -0,0 +1,184 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { PromptBoxAudio, usePromptAudioStore } from "@storyteller/ui-promptbox";
import {
UploadAudioMedia,
UploadImageMedia,
FilterMediaClasses,
} from "@storyteller/api";
import { useOmniGenAudioModels, useAudioCostEstimate } from "@storyteller/omni-gen";
import BackgroundGallery from "../PageImage/BackgroundGallery";
import {
galleryModalLightboxImage,
galleryModalLightboxMediaId,
galleryModalLightboxVisible,
galleryModalLightboxNavPrev,
galleryModalLightboxNavNext,
} from "@storyteller/ui-gallery-modal";
import { HelpMenuButton } from "@storyteller/ui-help-menu";
import {
useGalleryData,
useGenerationJobs,
type GalleryItem,
} from "@storyteller/ui-generation-list";
import { useDesktopUsername } from "~/components/generation-feed/useDesktopUsername";
import { DesktopCreatePageShell } from "~/components/generation-feed/DesktopCreatePageShell";
import { DesktopGenerationGallery } from "~/components/generation-feed/DesktopGenerationGallery";
const AUDIO_FILTER = [FilterMediaClasses.AUDIO];
const CreateAudio = () => {
const { models } = useOmniGenAudioModels();
const promptContentRef = useRef<HTMLDivElement>(null);
const [promptHeight, setPromptHeight] = useState<number>(138);
// Cost estimate inputs come from the shared audio prompt store (the
// promptbox owns the settings; the page only observes them).
const selectedModelId = usePromptAudioStore((s) => s.selectedModelId);
const referenceAudios = usePromptAudioStore((s) => s.referenceAudios);
const referenceImages = usePromptAudioStore((s) => s.referenceImages);
const selectedModel = useMemo(
() =>
models.find((m) => m.model === selectedModelId) ??
models.find((m) => m.model === "suno_music") ??
models[0],
[models, selectedModelId],
);
const audioCredits = useAudioCostEstimate({
model: selectedModel?.model ?? "",
audioReferenceCount: referenceAudios.length,
hasImageReference: referenceImages.length > 0,
sampleRateHz: selectedModel?.sample_rate_hz_options?.length
? (selectedModel.sample_rate_hz_default ?? undefined)
: undefined,
});
// Track the promptbox height so the feed can pad past it.
useEffect(() => {
const el = promptContentRef.current;
if (!el || typeof ResizeObserver === "undefined") return;
const update = () => setPromptHeight(el.offsetHeight);
update();
const ro = new ResizeObserver(() => update());
ro.observe(el);
return () => ro.disconnect();
}, []);
// The merged generation feed. Audio enqueues over HTTP (no Tauri task
// queue), so in-progress/failed come from the shared jobs poller and the
// completed history from the library — exactly like the webapp.
const username = useDesktopUsername();
const feed = useGenerationJobs({ mediaType: "audio", enabled: !!username });
const gallery = useGalleryData({
username,
filterMediaClasses: AUDIO_FILTER,
excludeUploads: true,
});
// Content only — while the gallery is still loading the shell keeps the
// hero + background up as a splash and fades them out when items land.
const hasContent =
feed.inProgress.length > 0 ||
feed.failed.length > 0 ||
feed.newlyCompleted.length > 0 ||
gallery.items.length > 0;
const newlyCompletedTokens = useMemo(
() => new Set(feed.newlyCompleted.map((item) => item.id)),
[feed.newlyCompleted],
);
// Flat, time-sorted completed list driving lightbox prev/next navigation.
const flatCompleted = useMemo(() => {
const seen = new Set<string>();
const merged: GalleryItem[] = [];
for (const item of feed.newlyCompleted) {
if (!seen.has(item.id)) {
seen.add(item.id);
merged.push(item);
}
}
for (const item of gallery.items) {
if (!seen.has(item.id)) {
seen.add(item.id);
merged.push(item);
}
}
merged.sort(
(a, b) =>
new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
);
return merged;
}, [feed.newlyCompleted, gallery.items]);
const flatCompletedRef = useRef(flatCompleted);
flatCompletedRef.current = flatCompleted;
// Open a completed row in the global lightbox (rendered by TopBar's
// gallery modal); prev/next walk the merged feed order.
const openInLightbox = useCallback((item: GalleryItem) => {
const list = flatCompletedRef.current;
const index = list.findIndex((i) => i.id === item.id);
galleryModalLightboxNavPrev.value =
index > 0 ? () => openInLightbox(list[index - 1]) : null;
galleryModalLightboxNavNext.value =
index >= 0 && index < list.length - 1
? () => openInLightbox(list[index + 1])
: null;
galleryModalLightboxMediaId.value = item.id;
galleryModalLightboxImage.value = {
id: item.id,
label: item.label,
thumbnail: item.thumbnail,
fullImage: item.fullImage,
createdAt: item.createdAt,
mediaClass: item.mediaClass,
};
galleryModalLightboxVisible.value = true;
}, []);
return (
<DesktopCreatePageShell
hasContent={hasContent}
emptyStateTitle="Create Audio"
emptyStateSubtitle="Describe a song, a sound, or a sample"
background={<BackgroundGallery />}
bottomOffset={promptHeight + 40}
listContent={
<DesktopGenerationGallery
inProgressJobs={feed.inProgress}
failedJobs={feed.failed}
onDismissFailed={feed.dismissFailed}
newlyCompletedItems={feed.newlyCompleted}
galleryItems={gallery.items}
newlyCompletedTokens={newlyCompletedTokens}
hasMore={gallery.hasMore}
isLoading={gallery.isLoading}
isInitialLoading={gallery.isInitialLoading}
onLoadMore={gallery.loadMore}
onGalleryItemClick={openInLightbox}
/>
}
promptBox={
<div className="fixed bottom-4 left-1/2 z-20 w-full max-w-5xl -translate-x-1/2 px-2 sm:px-4">
<div ref={promptContentRef}>
<PromptBoxAudio
models={models}
uploadAudio={UploadAudioMedia}
uploadImage={UploadImageMedia}
credits={audioCredits}
onEnqueuePressed={async () => {
// Nudge the shared jobs poller so the pending cards appear
// immediately (one per created job token).
window.dispatchEvent(new Event("task-queue-update"));
window.dispatchEvent(new Event("credits-change"));
}}
/>
</div>
</div>
}
bottomRight={<HelpMenuButton />}
/>
);
};
export default CreateAudio;
@@ -7,6 +7,7 @@ export type TabId =
| "VIDEO"
| "EDIT"
| "IMAGE"
| "AUDIO"
| "APPS"
| "VIDEO_FRAME_EXTRACTOR"
| "VIDEO_WATERMARK_REMOVAL"
-1
View File
@@ -75,7 +75,6 @@
"tailwind-merge": "^2.2.2",
"terser": "^5.30.3",
"uuid": "^9.0.1",
"wavesurfer.js": "^7.7.11",
"webm-writer": "^1.0.0"
},
"devDependencies": {
+3
View File
@@ -37,6 +37,9 @@
{
"path": "../../libs/components/vfx"
},
{
"path": "../../libs/omni-gen"
},
{
"path": "../../libs/components/slider-v2"
},
+152
View File
@@ -73,6 +73,89 @@ export interface OmniGenVideoGenerateResponse {
inference_job_token: string;
}
// ── Audio request / response types ───────────────────────────────────────
// Audio model ids (match backend CommonAudioModel).
export type OmniGenAudioModelId =
| "suno_music"
| "suno_remix"
| "suno_sounds"
| "suno_sample"
| "seed_audio_1p0";
// Musical keys for audio generation (eg. Suno Sounds).
// NB: There are intentionally no E keys, per product spec.
export type OmniGenMusicalKey =
| "auto"
| "c_major"
| "c_minor"
| "d_major"
| "d_minor"
| "f_major"
| "f_minor"
| "g_major"
| "g_minor"
| "a_major"
| "a_minor"
| "b_major"
| "b_minor";
/** Shared request body for both the audio cost estimate and audio
* generation endpoints. */
export interface OmniGenAudioRequest {
// REQUIRED (even if marked optional) — prevents duplicate requests.
idempotency_token?: string | null;
// REQUIRED (even if marked optional) — which model to use.
model?: OmniGenAudioModelId | string | null;
prompt?: string | null;
// Style/genre direction (Suno's "tags").
style_prompt?: string | null;
// The remix/sample source, or Seed Audio @Audio references (up to 3).
audio_media_tokens?: string[] | null;
// Seed Audio supports a single reference image; cannot be combined with
// audio references.
image_media_tokens?: string[] | null;
// Whether to keep the original lyrics (Suno Remix).
keep_lyrics?: boolean | null;
// Whether to generate instrumental-only audio (Suno Music / Sample).
is_instrumental?: boolean | null;
// Whether the sound should loop vs a single hit (Suno Sounds).
is_loopable?: boolean | null;
// Beats per minute (Suno Sounds).
bpm?: number | null;
// The musical key to use (Suno Sounds).
musical_key?: OmniGenMusicalKey | null;
// Output sample rate in Hz (Seed Audio: 8000/16000/24000/32000/44100/48000).
sample_rate_hz?: number | null;
// Playback speed multiplier (Seed Audio: 0.5–2.0).
speed?: number | null;
// Volume multiplier (Seed Audio: 0.5–2.0).
volume?: number | null;
// Pitch shift in semitones (Seed Audio: -12..=12).
pitch?: number | null;
}
export interface OmniGenAudioCostResponse {
success: boolean;
cost_in_credits?: number | null;
cost_in_usd_cents?: number | null;
// Whether failures are refunded. True: 100% yes. False: 100% no.
// Null/undefined: unknown or variable.
failures_are_refunded?: boolean | null;
has_watermark: boolean;
is_free: boolean;
is_rate_limited: boolean;
is_unlimited: boolean;
}
export interface OmniGenAudioGenerateResponse {
success: boolean;
inference_job_token: string;
// All job tokens created by this request (including the primary). One
// request may create multiple jobs (eg. Suno multi-clip) — poll them all.
all_job_tokens: string[];
}
// ── Image model info (from GET /v1/omni_gen/models/image) ────────────────
export interface OmniGenImageModelInfo {
@@ -168,6 +251,47 @@ export interface OmniGenVideoModelsResponse {
providers: OmniGenProviderEntry[];
}
// ── Audio model info (from GET /v1/omni_gen/models/audio) ────────────────
// Unlike the image/video model infos, absent capabilities are omitted from
// the JSON entirely (serde skip_serializing_if), so every field except
// `model` is optional. Treat undefined as unsupported.
export interface OmniGenAudioModelDetails {
model: OmniGenAudioModelId | string;
// ModelCreator enum as a snake_case string, e.g. "suno", "bytedance".
model_creator?: string | null;
full_name?: string | null;
extra_info?: string | null;
extra_info_short?: string | null;
text_prompt_supported?: boolean | null;
// Whether a style/genre prompt (Suno's "tags") is supported.
style_prompt_supported?: boolean | null;
audio_references_supported?: boolean | null;
audio_references_max?: number | null;
image_references_supported?: boolean | null;
image_references_max?: number | null;
// Whether the "keep lyrics" toggle is supported (Suno Remix).
keep_lyrics_supported?: boolean | null;
// Whether the instrumental-only toggle is supported (Suno Music / Sample).
instrumental_toggle_supported?: boolean | null;
// Whether the loop vs single-hit toggle is supported (Suno Sounds).
loopable_toggle_supported?: boolean | null;
bpm_supported?: boolean | null;
musical_key_supported?: boolean | null;
sample_rate_hz_options?: number[] | null;
sample_rate_hz_default?: number | null;
speed_supported?: boolean | null;
volume_supported?: boolean | null;
pitch_supported?: boolean | null;
is_disabled?: boolean | null;
}
export interface OmniGenAudioModelsResponse {
success: boolean;
models: OmniGenAudioModelDetails[];
providers: OmniGenProviderEntry[];
}
// ── Provider types (shared by image and video model responses) ───────────
export interface OmniGenProviderModelEntry {
@@ -213,6 +337,16 @@ export class OmniGenApi extends ApiManager {
});
}
public async getAudioModels(
provider?: string,
): Promise<OmniGenAudioModelsResponse> {
const query = provider ? { provider } : undefined;
return this.get<OmniGenAudioModelsResponse>({
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/models/audio`,
query,
});
}
// ── Cost estimates ───────────────────────────────────────────────────
public async estimateImageCost(
@@ -233,6 +367,15 @@ export class OmniGenApi extends ApiManager {
});
}
public async estimateAudioCost(
body: OmniGenAudioRequest,
): Promise<OmniGenAudioCostResponse> {
return this.post<Record<string, unknown>, OmniGenAudioCostResponse>({
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/cost/audio`,
body: stripNulls(body),
});
}
// ── Generation ───────────────────────────────────────────────────────
public async generateImage(
@@ -252,4 +395,13 @@ export class OmniGenApi extends ApiManager {
body: stripNulls(body),
});
}
public async generateAudio(
body: OmniGenAudioRequest,
): Promise<OmniGenAudioGenerateResponse> {
return this.post<Record<string, unknown>, OmniGenAudioGenerateResponse>({
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/generate/audio`,
body: stripNulls(body),
});
}
}
@@ -61,6 +61,12 @@ export function getMediaThumbnail(
let thumbnailUrl: string | null = null;
if (mediaClass === "audio") {
// Audio has no image thumbnail; falling through to cdn_url would hand an
// mp3 to an <img>. Callers render a placeholder/waveform instead.
return null;
}
if (mediaClass === "video" && mediaLinks.maybe_video_previews?.animated) {
thumbnailUrl = mediaLinks.maybe_video_previews.animated;
} else if (mediaLinks.maybe_thumbnail_template) {
@@ -0,0 +1,20 @@
{
"name": "@storyteller/ui-audio-player",
"version": "0.0.1",
"type": "module",
"main": "./dist/index.js",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
"./package.json": "./package.json",
".": {
"development": "./src/index.ts",
"types": "./dist/index.d.ts",
"import": "./dist/index.js",
"default": "./dist/index.js"
}
},
"dependencies": {
"wavesurfer.js": "^7.7.11"
}
}
@@ -0,0 +1,2 @@
export * from "./lib/WaveformAudioPlayer.js";
export * from "./lib/audio-playback-controller.js";
@@ -0,0 +1,369 @@
import {
MouseEvent,
useCallback,
useEffect,
useId,
useMemo,
useRef,
useState,
} from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faPause,
faPlay,
faVolume,
faVolumeSlash,
} from "@fortawesome/pro-solid-svg-icons";
import WaveSurfer from "wavesurfer.js";
import { addCorsParam } from "@storyteller/common";
import { SliderV2 } from "@storyteller/ui-sliderv2";
import {
notifyAudioStopped,
registerAudioPlayer,
requestAudioPlayback,
unregisterAudioPlayer,
} from "./audio-playback-controller.js";
const WAVE_COLOR = "rgba(255, 255, 255, 0.35)";
// Tailwind `primary` (#2d81ff) — canvas fillStyle can't resolve CSS vars, so
// the hex is inlined. Keep in sync with the apps' tailwind primary color.
const WAVE_PROGRESS_COLOR = "#2d81ff";
const PLACEHOLDER_BAR_COUNT = 48;
// Decoded waveforms are expensive (full fetch + AudioContext decode), so we
// keep them per URL for the lifetime of the session. Re-mounts (view toggles,
// lightbox, scroll-recycling) reuse the peaks instead of re-downloading.
const peaksCache = new Map<string, { peaks: number[][]; duration: number }>();
export interface WaveformAudioPlayerProps {
src: string;
// Known duration (e.g. MediaFile.maybe_duration_millis) shown before the
// audio metadata loads.
durationMillis?: number | null;
// Tighter layout for list rows.
compact?: boolean;
className?: string;
}
export function WaveformAudioPlayer({
src,
durationMillis,
compact = false,
className = "",
}: WaveformAudioPlayerProps) {
const playerId = useId();
const waveContainerRef = useRef<HTMLDivElement | null>(null);
const rootRef = useRef<HTMLDivElement | null>(null);
const audioRef = useRef<HTMLAudioElement | null>(null);
const waveSurferRef = useRef<WaveSurfer | null>(null);
const [isPlaying, setIsPlaying] = useState(false);
const [currentTime, setCurrentTime] = useState(0);
const [duration, setDuration] = useState(
durationMillis ? durationMillis / 1000 : 0,
);
// "pending" until the card scrolls into view, then "ready" once wavesurfer
// has drawn, or "failed" if fetch/decode errored (CORS etc.) — playback
// still works through the bare <audio> element in that case.
const [waveState, setWaveState] = useState<"pending" | "ready" | "failed">(
"pending",
);
const [volume, setVolume] = useState(100);
const [showVolume, setShowVolume] = useState(false);
const waveHeight = compact ? 32 : 44;
// ── Audio element (owns playback in both wave and fallback modes) ──────
useEffect(() => {
const audio = new Audio();
audio.preload = "metadata";
audio.src = src;
audioRef.current = audio;
const onPlay = () => {
setIsPlaying(true);
requestAudioPlayback(playerId);
};
const onPause = () => {
setIsPlaying(false);
notifyAudioStopped(playerId);
};
const onTimeUpdate = () => setCurrentTime(audio.currentTime);
const onLoadedMetadata = () => {
if (Number.isFinite(audio.duration) && audio.duration > 0) {
setDuration(audio.duration);
}
};
audio.addEventListener("play", onPlay);
audio.addEventListener("pause", onPause);
audio.addEventListener("ended", onPause);
audio.addEventListener("timeupdate", onTimeUpdate);
audio.addEventListener("loadedmetadata", onLoadedMetadata);
registerAudioPlayer(playerId, () => audio.pause());
return () => {
unregisterAudioPlayer(playerId);
audio.pause();
audio.removeEventListener("play", onPlay);
audio.removeEventListener("pause", onPause);
audio.removeEventListener("ended", onPause);
audio.removeEventListener("timeupdate", onTimeUpdate);
audio.removeEventListener("loadedmetadata", onLoadedMetadata);
audio.removeAttribute("src");
audio.load();
audioRef.current = null;
};
}, [src, playerId]);
// ── Wavesurfer visualization (lazy: only once scrolled into view) ──────
useEffect(() => {
const root = rootRef.current;
if (!root) return;
let cancelled = false;
const initWaveSurfer = () => {
const container = waveContainerRef.current;
const media = audioRef.current;
if (cancelled || !container || !media || waveSurferRef.current) return;
const cached = peaksCache.get(src);
try {
const ws = WaveSurfer.create({
container,
media,
// The waveform decode goes through fetch(), which needs CORS
// headers from the CDN; playback (the media element) does not.
url: addCorsParam(src) ?? src,
height: waveHeight,
waveColor: WAVE_COLOR,
progressColor: WAVE_PROGRESS_COLOR,
cursorColor: "transparent",
barWidth: 2,
barGap: 1.5,
barRadius: 2,
normalize: true,
interact: true,
...(cached
? { peaks: cached.peaks, duration: cached.duration }
: {}),
});
waveSurferRef.current = ws;
ws.on("decode", (decodedDuration: number) => {
if (cancelled) return;
setWaveState("ready");
if (decodedDuration > 0) setDuration(decodedDuration);
if (!peaksCache.has(src)) {
try {
peaksCache.set(src, {
peaks: ws.exportPeaks({ maxLength: 1024 }),
duration: decodedDuration,
});
} catch {
// Peaks export is a cache optimization only — safe to skip.
}
}
});
ws.on("error", () => {
if (cancelled) return;
setWaveState("failed");
waveSurferRef.current = null;
try {
ws.destroy();
} catch {
// Already torn down.
}
});
} catch {
if (!cancelled) setWaveState("failed");
}
};
if (typeof IntersectionObserver === "undefined") {
initWaveSurfer();
return () => {
cancelled = true;
destroyWaveSurfer(waveSurferRef);
};
}
const observer = new IntersectionObserver(
(entries) => {
if (entries.some((entry) => entry.isIntersecting)) {
observer.disconnect();
initWaveSurfer();
}
},
{ threshold: 0.15 },
);
observer.observe(root);
return () => {
cancelled = true;
observer.disconnect();
destroyWaveSurfer(waveSurferRef);
};
}, [src, waveHeight]);
// ── Handlers ────────────────────────────────────────────────────────────
const togglePlay = useCallback(() => {
const audio = audioRef.current;
if (!audio) return;
if (audio.paused) {
void audio.play().catch(() => setIsPlaying(false));
} else {
audio.pause();
}
}, []);
const handleVolumeChange = useCallback((value: number) => {
setVolume(value);
const audio = audioRef.current;
if (audio) audio.volume = value / 100;
}, []);
// Seek on the placeholder / fallback progress bar (the real waveform has
// built-in click-to-seek via wavesurfer).
const handleBarSeek = useCallback(
(event: MouseEvent<HTMLDivElement>) => {
const audio = audioRef.current;
if (!audio || !duration) return;
const rect = event.currentTarget.getBoundingClientRect();
const ratio = (event.clientX - rect.left) / rect.width;
audio.currentTime = Math.max(0, Math.min(1, ratio)) * duration;
},
[duration],
);
// Close the volume flyout on outside clicks.
useEffect(() => {
if (!showVolume) return;
const onDocClick = (event: Event) => {
if (!rootRef.current?.contains(event.target as Node)) {
setShowVolume(false);
}
};
document.addEventListener("mousedown", onDocClick);
return () => document.removeEventListener("mousedown", onDocClick);
}, [showVolume]);
const placeholderBars = useMemo(() => buildPlaceholderBars(src), [src]);
const progressRatio = duration > 0 ? currentTime / duration : 0;
const showWave = waveState !== "failed";
return (
<div
ref={rootRef}
className={`flex items-center gap-2.5 ${compact ? "px-2.5 py-2" : "px-3 py-2.5"} ${className}`}
onClick={(event) => event.stopPropagation()}
>
<button
type="button"
aria-label={isPlaying ? "Pause" : "Play"}
onClick={togglePlay}
className={`flex shrink-0 items-center justify-center rounded-full bg-white text-black transition-transform hover:scale-105 active:scale-95 ${compact ? "h-8 w-8" : "h-10 w-10"}`}
>
<FontAwesomeIcon
icon={isPlaying ? faPause : faPlay}
className={`${compact ? "h-3 w-3" : "h-3.5 w-3.5"} ${isPlaying ? "" : "ml-0.5"}`}
/>
</button>
<div className="relative min-w-0 flex-1" style={{ height: waveHeight }}>
{showWave && (
<div ref={waveContainerRef} className="absolute inset-0" />
)}
{(waveState === "pending" || waveState === "failed") && (
<div
className="absolute inset-0 flex cursor-pointer items-center gap-[1.5px]"
onClick={handleBarSeek}
>
{waveState === "failed" ? (
<div className="h-1.5 w-full overflow-hidden rounded-full bg-white/20">
<div
className="h-full rounded-full bg-primary"
style={{ width: `${progressRatio * 100}%` }}
/>
</div>
) : (
placeholderBars.map((barHeight, index) => (
<div
key={index}
className="w-[2px] shrink-0 rounded-full bg-white/25"
style={{ height: `${barHeight}%` }}
/>
))
)}
</div>
)}
</div>
<span
className={`shrink-0 tabular-nums text-white/70 ${compact ? "text-[11px]" : "text-xs"}`}
>
{formatTime(currentTime)} / {duration > 0 ? formatTime(duration) : "-:--"}
</span>
<div className="relative shrink-0">
<button
type="button"
aria-label="Volume"
onClick={() => setShowVolume((visible) => !visible)}
className="flex h-7 w-7 items-center justify-center rounded-md text-white/70 transition-colors hover:bg-white/10 hover:text-white"
>
<FontAwesomeIcon
icon={volume === 0 ? faVolumeSlash : faVolume}
className="h-3.5 w-3.5"
/>
</button>
{showVolume && (
<div className="absolute bottom-full right-0 z-20 mb-2 w-28 rounded-lg border border-white/10 bg-neutral-900/95 px-3 py-2.5 shadow-xl backdrop-blur">
<SliderV2
min={0}
max={100}
step={1}
value={volume}
onChange={handleVolumeChange}
variant="filled"
/>
</div>
)}
</div>
</div>
);
}
function destroyWaveSurfer(ref: { current: WaveSurfer | null }): void {
if (!ref.current) return;
try {
ref.current.destroy();
} catch {
// Already torn down.
}
ref.current = null;
}
function formatTime(totalSeconds: number): string {
const seconds = Math.max(0, Math.floor(totalSeconds));
const minutes = Math.floor(seconds / 60);
const remainder = seconds % 60;
return `${minutes}:${remainder.toString().padStart(2, "0")}`;
}
// Deterministic pseudo-waveform so unloaded cards still read as audio and
// don't all look identical. Seeded by the URL, not random — stable renders.
function buildPlaceholderBars(seedSource: string): number[] {
let seed = 0;
for (let index = 0; index < seedSource.length; index++) {
seed = (seed * 31 + seedSource.charCodeAt(index)) >>> 0;
}
const bars: number[] = [];
for (let index = 0; index < PLACEHOLDER_BAR_COUNT; index++) {
seed = (seed * 1103515245 + 12345) >>> 0;
bars.push(25 + (seed % 65));
}
return bars;
}
@@ -0,0 +1,61 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
notifyAudioStopped,
registerAudioPlayer,
requestAudioPlayback,
resetAudioPlaybackControllerForTests,
unregisterAudioPlayer,
} from "./audio-playback-controller";
describe("audio-playback-controller", () => {
beforeEach(() => {
resetAudioPlaybackControllerForTests();
});
it("pauses the previously playing player when another starts", () => {
const pauseA = vi.fn();
const pauseB = vi.fn();
registerAudioPlayer("a", pauseA);
registerAudioPlayer("b", pauseB);
requestAudioPlayback("a");
expect(pauseA).not.toHaveBeenCalled();
requestAudioPlayback("b");
expect(pauseA).toHaveBeenCalledTimes(1);
expect(pauseB).not.toHaveBeenCalled();
});
it("does not pause a player that re-requests playback (seek/resume)", () => {
const pauseA = vi.fn();
registerAudioPlayer("a", pauseA);
requestAudioPlayback("a");
requestAudioPlayback("a");
expect(pauseA).not.toHaveBeenCalled();
});
it("clears the current player when it stops on its own", () => {
const pauseA = vi.fn();
registerAudioPlayer("a", pauseA);
requestAudioPlayback("a");
notifyAudioStopped("a");
// A new player starting should not pause the already-stopped one.
const pauseB = vi.fn();
registerAudioPlayer("b", pauseB);
requestAudioPlayback("b");
expect(pauseA).not.toHaveBeenCalled();
});
it("unregistering the current player forgets it", () => {
const pauseA = vi.fn();
registerAudioPlayer("a", pauseA);
requestAudioPlayback("a");
unregisterAudioPlayer("a");
requestAudioPlayback("b");
expect(pauseA).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,40 @@
// Global playback coordinator: ensures only one audio player is audible at a
// time. Players register a pause callback; when one starts playing it calls
// requestAudioPlayback, which pauses whichever player was playing before it.
type PauseFn = () => void;
let currentId: string | null = null;
const players = new Map<string, PauseFn>();
export function registerAudioPlayer(id: string, pause: PauseFn): void {
players.set(id, pause);
}
export function unregisterAudioPlayer(id: string): void {
players.delete(id);
if (currentId === id) {
currentId = null;
}
}
/** Call when a player starts playing. Pauses the previously playing player. */
export function requestAudioPlayback(id: string): void {
if (currentId && currentId !== id) {
players.get(currentId)?.();
}
currentId = id;
}
/** Call when a player stops on its own (pause/finish/unmount). */
export function notifyAudioStopped(id: string): void {
if (currentId === id) {
currentId = null;
}
}
/** Test helper: reset module state between test cases. */
export function resetAudioPlaybackControllerForTests(): void {
currentId = null;
players.clear();
}
@@ -0,0 +1,19 @@
{
"files": [],
"include": [],
"references": [
{
"path": "../slider-v2"
},
{
"path": "../../common"
},
{
"path": "./tsconfig.lib.json"
},
{
"path": "./tsconfig.spec.json"
}
],
"extends": "../../../tsconfig.base.json"
}
@@ -0,0 +1,53 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"types": [
"node",
"@nx/react/typings/cssmodule.d.ts",
"@nx/react/typings/image.d.ts",
"vite/client"
],
"rootDir": "src",
"jsx": "react-jsx",
"tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo"
},
"exclude": [
"out-tsc",
"dist",
"**/*.spec.ts",
"**/*.test.ts",
"**/*.spec.tsx",
"**/*.test.tsx",
"**/*.spec.js",
"**/*.test.js",
"**/*.spec.jsx",
"**/*.test.jsx",
"vite.config.ts",
"vite.config.mts",
"vitest.config.ts",
"vitest.config.mts",
"src/**/*.test.ts",
"src/**/*.spec.ts",
"src/**/*.test.tsx",
"src/**/*.spec.tsx",
"src/**/*.test.js",
"src/**/*.spec.js",
"src/**/*.test.jsx",
"src/**/*.spec.jsx"
],
"include": [
"src/**/*.js",
"src/**/*.jsx",
"src/**/*.ts",
"src/**/*.tsx"
],
"references": [
{
"path": "../slider-v2/tsconfig.lib.json"
},
{
"path": "../../common/tsconfig.lib.json"
}
]
}
@@ -0,0 +1,34 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "./out-tsc/vitest",
"types": [
"vitest/globals",
"vitest/importMeta",
"vite/client",
"node",
"vitest"
],
"jsx": "react-jsx"
},
"include": [
"vite.config.ts",
"vite.config.mts",
"vitest.config.ts",
"vitest.config.mts",
"src/**/*.test.ts",
"src/**/*.spec.ts",
"src/**/*.test.tsx",
"src/**/*.spec.tsx",
"src/**/*.test.js",
"src/**/*.spec.js",
"src/**/*.test.jsx",
"src/**/*.spec.jsx",
"src/**/*.d.ts"
],
"references": [
{
"path": "./tsconfig.lib.json"
}
]
}
@@ -0,0 +1,41 @@
/// <reference types='vitest' />
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
import dts from 'vite-plugin-dts';
import * as path from 'path';
import { isExternal } from '../../shared-vite-config';
export default defineConfig(() => ({
root: __dirname,
cacheDir: '../../../node_modules/.vite/libs/components/audio-player',
plugins: [react(), dts({ entryRoot: 'src', tsconfigPath: path.join(__dirname, 'tsconfig.lib.json') })],
build: {
outDir: './dist',
emptyOutDir: true,
reportCompressedSize: true,
commonjsOptions: {
transformMixedEsModules: true,
},
lib: {
entry: 'src/index.ts',
name: '@storyteller/ui-audio-player',
fileName: 'index',
formats: ['es' as const]
},
rollupOptions: {
// External packages that should not be bundled into your library.
external: isExternal
},
},
test: {
watch: false,
globals: true,
environment: 'jsdom',
include: ['{src,tests}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
reporters: ['default'],
coverage: {
reportsDirectory: './test-output/vitest/coverage',
provider: 'v8' as const,
}
},
}));
@@ -26,8 +26,10 @@ export const ToggleButton = ({
return (
<Button
className={twMerge(
"flex h-9 items-center justify-center rounded-lg border-2 border-transparent text-sm text-white backdrop-blur-lg transition-all",
hasLabel ? "px-3" : "w-9",
// 34px matches the sibling toolbar controls (GenerateButton, the
// PopoverMenu triggers); the 2px border is inside the box.
"flex h-[34px] items-center justify-center rounded-lg border-2 border-transparent text-sm text-white backdrop-blur-lg transition-all",
hasLabel ? "px-3" : "w-[34px]",
isActive
? "border-white/20 bg-brand-primary/40 hover:border-white/30 hover:bg-brand-primary/40"
: "bg-[#5F5F68]/60 hover:bg-[#5F5F68]/90",
@@ -7,6 +7,7 @@ export * from "./lib/gallery-view-store";
export * from "./lib/GalleryViewToggle";
export * from "./lib/useGalleryData";
export * from "./lib/useGalleryEntries";
export * from "./lib/useGenerationJobs";
export * from "./lib/GalleryThumbnail";
export * from "./lib/CopyPromptButton";
export * from "./lib/PendingRow";
@@ -1,10 +1,16 @@
import { memo, useCallback, useState, type ReactNode } from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faCube, faImage, faVideo } from "@fortawesome/pro-solid-svg-icons";
import {
faCube,
faImage,
faMusic,
faVideo,
} from "@fortawesome/pro-solid-svg-icons";
import {
getCreatorIconPathForModelId,
getModelDisplayName,
} from "@storyteller/model-list";
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
import { GalleryThumbnail } from "./GalleryThumbnail";
import type { GalleryItem } from "./types";
@@ -15,6 +21,9 @@ const STORAGE_KEY = "gallery-aspect-ratios";
// Cap ratio so tall portraits don't dominate — 1.4 ≈ 5:7
const MAX_RATIO = 1.4;
// Audio cards have no image to measure — fixed square tile.
const AUDIO_RATIO = 1;
function loadCache(): Map<string, number> {
const map = new Map<string, number>();
try {
@@ -59,6 +68,9 @@ export interface GalleryCardProps {
// "auto" = dynamic aspect ratio from the loaded image (masonry layouts).
// "square" = fixed 1:1; skips the ratio measurement path (uniform grids).
shape?: "auto" | "square";
/** Prompt text resolved by the view (via the prompts cache). Audio cards
* show it inline since they have no image to speak for themselves. */
title?: string;
/** Hover-revealed quick-action cluster (recreate / share / download …). */
actionsSlot?: ReactNode;
}
@@ -67,6 +79,7 @@ export const GalleryCard = memo(function GalleryCard({
item,
onClick,
shape = "auto",
title,
actionsSlot,
}: GalleryCardProps) {
const isSquare = shape === "square";
@@ -75,8 +88,15 @@ export const GalleryCard = memo(function GalleryCard({
const isVideo = item.mediaClass === "video";
const is3D = item.mediaClass === "dimensional";
const mediaIcon = isVideo ? faVideo : is3D ? faCube : faImage;
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : "Image";
const isAudio = item.mediaClass === "audio";
const mediaIcon = isVideo
? faVideo
: is3D
? faCube
: isAudio
? faMusic
: faImage;
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : isAudio ? "Audio" : "Image";
const modelDisplayName = item.modelId
? getModelDisplayName(item.modelId)
: null;
@@ -84,7 +104,11 @@ export const GalleryCard = memo(function GalleryCard({
? getCreatorIconPathForModelId(item.modelId)
: null;
const displayRatio = ratio ? Math.min(ratio, MAX_RATIO) : 1;
const displayRatio = isAudio
? AUDIO_RATIO
: ratio
? Math.min(ratio, MAX_RATIO)
: 1;
// In square mode the wrapper sets the ratio via `aspect-square`; we only
// compute the dynamic aspectRatio for masonry-style layouts.
@@ -136,13 +160,42 @@ export const GalleryCard = memo(function GalleryCard({
className="absolute inset-0 overflow-hidden rounded-[inherit]"
style={{ contentVisibility: "auto", containIntrinsicSize: "auto 200px" }}
>
<GalleryThumbnail
thumbnail={item.thumbnail}
alt={item.label}
isVideo={isVideo}
fallbackIcon={mediaIcon}
onLoad={measureRatio}
/>
{isAudio ? (
<div className="flex h-full flex-col bg-gradient-to-br from-white/[0.07] to-white/[0.02] p-3">
<div className="flex items-start gap-2">
<p className="line-clamp-3 min-w-0 flex-1 text-sm leading-snug text-white/85">
{title || item.label}
</p>
<FontAwesomeIcon
icon={faMusic}
className="pointer-events-none mt-0.5 shrink-0 text-base text-white/15"
/>
</div>
{item.fullImage ? (
<div className="flex min-h-0 flex-1 items-center">
<div className="w-full">
<WaveformAudioPlayer
src={item.fullImage}
durationMillis={item.durationMillis}
/>
</div>
</div>
) : (
<FontAwesomeIcon
icon={faMusic}
className="m-auto text-2xl text-white/20"
/>
)}
</div>
) : (
<GalleryThumbnail
thumbnail={item.thumbnail}
alt={item.label}
isVideo={isVideo}
fallbackIcon={mediaIcon}
onLoad={measureRatio}
/>
)}
</div>
{/* Hover overlay with media type + model badges and quick actions */}
@@ -1,10 +1,17 @@
import { memo, useCallback, type ReactNode } from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faCube, faImage, faPlay, faVideo } from "@fortawesome/pro-solid-svg-icons";
import {
faCube,
faImage,
faMusic,
faPlay,
faVideo,
} from "@fortawesome/pro-solid-svg-icons";
import {
getCreatorIconPathForModelId,
getModelDisplayName,
} from "@storyteller/model-list";
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
import { GalleryThumbnail } from "./GalleryThumbnail";
import { CopyPromptButton } from "./CopyPromptButton";
import { formatTimeAgo } from "./format-time-ago";
@@ -35,8 +42,15 @@ export const GalleryRow = memo(function GalleryRow({
}: GalleryRowProps) {
const isVideo = item.mediaClass === "video";
const is3D = item.mediaClass === "dimensional";
const mediaIcon = isVideo ? faVideo : is3D ? faCube : faImage;
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : "Image";
const isAudio = item.mediaClass === "audio";
const mediaIcon = isVideo
? faVideo
: is3D
? faCube
: isAudio
? faMusic
: faImage;
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : isAudio ? "Audio" : "Image";
const effectiveModelId = modelId ?? item.modelId;
const modelDisplayName = effectiveModelId
@@ -108,6 +122,15 @@ export const GalleryRow = memo(function GalleryRow({
)}
</div>
)}
{isAudio && item.fullImage && (
<div className="mt-1.5 max-w-md rounded-md bg-white/[0.04]">
<WaveformAudioPlayer
src={item.fullImage}
durationMillis={item.durationMillis}
compact
/>
</div>
)}
<div className="mt-1 flex flex-wrap items-center gap-x-1.5 gap-y-1 text-xs text-white/45">
{modelIconPath && (
<img
@@ -1,4 +1,4 @@
import type { ReactNode } from "react";
import { useCallback, useMemo, type ReactNode } from "react";
import Masonry from "react-masonry-css";
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
import { PendingCard } from "./PendingCard";
@@ -8,6 +8,8 @@ import {
useMergedGalleryEntries,
useInfiniteScrollSentinel,
} from "./useGalleryEntries";
import { usePrompts } from "./prompts-cache";
import { useMediaPromptTokens } from "./media-prompt-token-cache";
import type { RecreateSlotContext } from "./GenerationListView";
import type { FailedJob, GalleryItem, InProgressJob } from "./types";
@@ -72,6 +74,51 @@ export function GenerationGridView({
newlyCompletedTokens,
});
// Audio cards show their prompt inline (no image speaks for them), so
// resolve prompt text the same way the list view does — but only for audio
// items, keeping image/video walls free of the extra lookups. Both steps
// (media token → prompt token → prompt text) are batched + cached.
const audioMediaTokensNeedingPrompt = useMemo(
() =>
mergedEntries.flatMap((entry) =>
entry.kind === "gallery" &&
entry.item.mediaClass === "audio" &&
!entry.item.promptToken
? [entry.item.id]
: [],
),
[mergedEntries],
);
const resolvedPromptTokens = useMediaPromptTokens(
audioMediaTokensNeedingPrompt,
);
const audioPromptTokens = useMemo(() => {
const tokens: string[] = [];
for (const entry of mergedEntries) {
if (entry.kind !== "gallery" || entry.item.mediaClass !== "audio") {
continue;
}
const token =
entry.item.promptToken || resolvedPromptTokens.get(entry.item.id);
if (token) tokens.push(token);
}
return tokens;
}, [mergedEntries, resolvedPromptTokens]);
const promptsMap = usePrompts(audioPromptTokens);
const audioTitleFor = useCallback(
(item: GalleryItem): string | undefined => {
if (item.mediaClass !== "audio") return undefined;
const token = item.promptToken || resolvedPromptTokens.get(item.id);
return (
(token ? promptsMap.get(token)?.maybe_positive_prompt : undefined) ||
undefined
);
},
[resolvedPromptTokens, promptsMap],
);
if (isInitialLoading) {
return (
<div className="flex justify-center py-20">
@@ -135,6 +182,7 @@ export function GenerationGridView({
<GalleryCard
item={entry.item}
onClick={onGalleryItemClick}
title={audioTitleFor(entry.item)}
actionsSlot={renderGalleryActions?.(entry.item, {
modelId: entry.item.modelId,
promptToken: entry.item.promptToken,
@@ -9,7 +9,12 @@ import {
} from "./useGalleryEntries";
import { usePrompts } from "./prompts-cache";
import { useMediaPromptTokens } from "./media-prompt-token-cache";
import type { FailedJob, GalleryItem, InProgressJob } from "./types";
import type {
FailedJob,
GalleryItem,
GenerationMediaClass,
InProgressJob,
} from "./types";
// Constrains the feed to the promptbox width (max-w-5xl) and stacks one row
// per generation: a single time-sorted merge of the in-progress / failed /
@@ -19,7 +24,7 @@ import type { FailedJob, GalleryItem, InProgressJob } from "./types";
export interface RecreateSlotContext {
promptToken: string;
mediaClass: "image" | "video";
mediaClass: GenerationMediaClass;
kind: "pending" | "failed";
}
@@ -3,6 +3,8 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faSpinnerThird } from "@fortawesome/pro-solid-svg-icons";
import { getCreatorIconPathForModelId } from "@storyteller/model-list";
import { derivePendingStatus } from "./pending-status";
import { batchNoun } from "./types";
import type { GenerationMediaClass } from "./types";
export interface PendingCardProps {
id: string;
@@ -12,7 +14,7 @@ export interface PendingCardProps {
progress?: number;
estimatedTimeLeftMs?: number;
batchCount?: number;
mediaClass: "image" | "video";
mediaClass: GenerationMediaClass;
/** Hover-revealed action (e.g. a Recreate button) next to the prompt. */
recreateSlot?: ReactNode;
}
@@ -39,8 +41,7 @@ export const PendingCard = memo(function PendingCard({
<div className="animate-shimmer h-full w-full" />
{batchCount != null && batchCount > 1 && (
<div className="absolute left-2 right-2 top-2 z-10 rounded-md bg-black/60 px-2.5 py-1.5 text-center text-[10px] leading-snug text-white/70 backdrop-blur-sm">
Generating {batchCount}{" "}
{mediaClass === "image" ? "images" : "videos"} · Results may appear
Generating {batchCount} {batchNoun(mediaClass)} · Results may appear
one at a time
</div>
)}
@@ -4,6 +4,8 @@ import { faSpinnerThird } from "@fortawesome/pro-solid-svg-icons";
import { getCreatorIconPathForModelId } from "@storyteller/model-list";
import { CopyPromptButton } from "./CopyPromptButton";
import { derivePendingStatus } from "./pending-status";
import { batchNoun } from "./types";
import type { GenerationMediaClass } from "./types";
export interface PendingRowProps {
id: string;
@@ -13,7 +15,7 @@ export interface PendingRowProps {
progress?: number;
estimatedTimeLeftMs?: number;
batchCount?: number;
mediaClass: "image" | "video";
mediaClass: GenerationMediaClass;
/** Hover-revealed action (e.g. a Recreate button) after the model label. */
recreateSlot?: ReactNode;
onCopyPromptResult?: (success: boolean) => void;
@@ -78,7 +80,7 @@ export const PendingRow = memo(function PendingRow({
<>
<span className="text-white/25">·</span>
<span className="shrink-0">
{batchCount} {mediaClass === "image" ? "images" : "videos"}
{batchCount} {batchNoun(mediaClass)}
</span>
</>
)}
@@ -1,6 +1,20 @@
// Canonical data shapes for the merged generation feed (in-progress / failed /
// completed). Hosts (webapp, desktop) map their own job sources into these.
export type GenerationMediaClass = "image" | "video" | "audio";
/** Plural noun for batch captions, e.g. "Generating 4 images". */
export function batchNoun(mediaClass: GenerationMediaClass): string {
switch (mediaClass) {
case "video":
return "videos";
case "audio":
return "audio clips";
default:
return "images";
}
}
export interface GalleryItem {
id: string;
label: string;
@@ -13,6 +27,8 @@ export interface GalleryItem {
// Token for the generation's prompt record. The list view resolves it
// (via the shared prompts cache) to show the real prompt + model.
promptToken?: string;
// Playback length for audio (and video) items, when the API knows it.
durationMillis?: number;
}
export interface InProgressJob {
@@ -27,7 +43,7 @@ export interface InProgressJob {
// Prompt token + media class enable the "Recreate" action while the job is
// still running, mirroring the failed/completed cards.
promptToken?: string;
mediaClass: "image" | "video";
mediaClass: GenerationMediaClass;
}
export interface FailedJob {
@@ -41,5 +57,5 @@ export interface FailedJob {
createdAt: string;
promptToken?: string;
refImageUrl?: string;
mediaClass: "image" | "video";
mediaClass: GenerationMediaClass;
}
@@ -16,6 +16,8 @@ const getLabel = (item: any) => {
return "Image Generation";
case "video":
return "Video Generation";
case "audio":
return "Audio Generation";
case "dimensional":
return "3D Mesh";
default:
@@ -42,12 +44,15 @@ export function useGalleryData(options: {
const api = useMemo(() => new GalleryModalApi(), []);
const mapApiItem = useCallback((item: any): GalleryItem => {
const isDimensional = item.media_class === "dimensional";
const thumbnail = isDimensional
? null
: getMediaThumbnail(item.media_links, item.media_class, {
// 3D and audio media have no image thumbnail — their cards render a
// placeholder icon / waveform player instead.
const hasImageThumbnail =
item.media_class !== "dimensional" && item.media_class !== "audio";
const thumbnail = hasImageThumbnail
? getMediaThumbnail(item.media_links, item.media_class, {
size: THUMBNAIL_SIZES.LARGE,
});
})
: null;
return {
id: item.token,
@@ -59,6 +64,7 @@ export function useGalleryData(options: {
modelId: item.maybe_model_type || undefined,
batchImageToken: item.maybe_batch_token,
promptToken: item.maybe_prompt_token || undefined,
durationMillis: item.maybe_duration_millis ?? undefined,
};
}, []);
@@ -0,0 +1,428 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { JobsApi, JobStatus, MediaFilesApi } from "@storyteller/api";
import type { Job, Prompts } from "@storyteller/api";
import { getMediaThumbnail, THUMBNAIL_SIZES } from "@storyteller/common";
import {
getModelDisplayName,
getProviderDisplayName,
ALL_MODELS_LIST,
} from "@storyteller/model-list";
import { getCachedPrompt, usePrompts } from "./prompts-cache";
import type {
FailedJob,
GalleryItem,
GenerationMediaClass,
InProgressJob,
} from "./types";
// Maps JobsApi's recent-jobs stream into the canonical feed shapes
// (InProgressJob / FailedJob / newly-completed GalleryItem). HTTP-based, so
// both the webapp and the desktop app can use it (the desktop image/video
// pages use the Tauri task queue instead; audio enqueues over HTTP).
// ── Constants ──────────────────────────────────────────────────────────────
const IN_PROGRESS_STATUSES = new Set([JobStatus.PENDING, JobStatus.STARTED]);
const COMPLETED_STATUSES = new Set([JobStatus.COMPLETE_SUCCESS]);
const FAILED_STATUSES = new Set([
JobStatus.ATTEMPT_FAILED,
JobStatus.COMPLETE_FAILURE,
JobStatus.DEAD,
JobStatus.CANCELLED_BY_USER,
JobStatus.CANCELLED_BY_SYSTEM,
]);
const FAILURE_REASON_LABEL: Record<string, string> = {
rule_bans_user_image: "Image violates content policy",
rule_bans_user_image_with_faces: "Images with faces are not allowed",
rule_bans_user_text_prompt: "Text prompt violates content policy",
rule_bans_user_content: "Content violates content policy",
rule_bans_generated_video: "Generated video flagged by content policy",
rule_bans_generated_audio: "Generated audio flagged by content policy",
rule_bans_generated_content: "Generated content flagged by content policy",
generation_failed: "Generation failed",
unknown: "An unknown error occurred",
};
// Progress-bar defaults when the model doesn't declare a progressBarTime.
// Mirrors the backend's estimate baselines (audio 120s, video 300s+).
const DEFAULT_DURATION_MS: Record<GenerationMediaClass, number> = {
image: 30000,
video: 900000,
audio: 120000,
};
// Cache per-task durations
const taskDurationCache = new Map<string, number>();
// ── Helpers ────────────────────────────────────────────────────────────────
function getJobMediaType(
job: Job,
): "image" | "video" | "audio" | "other" {
const cat = job.request.inference_category?.toLowerCase() ?? "";
if (cat.includes("video")) return "video";
if (cat.includes("image")) return "image";
if (cat.includes("audio")) return "audio";
return "other";
}
function getJobMediaClass(job: Job): GenerationMediaClass {
const inferred = getJobMediaType(job);
return inferred === "other" ? "image" : inferred;
}
function getModelLabel(job: Job, promptsMap?: Map<string, Prompts>): string {
const promptToken = job.request.maybe_prompt_token;
const cachedPrompt = promptToken
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
: undefined;
const modelType =
cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
const providerKey = cachedPrompt?.maybe_generation_provider ?? modelType;
const modelDisplay = modelType ? getModelDisplayName(modelType) : undefined;
const provider = providerKey
? getProviderDisplayName(providerKey.toLowerCase())
: undefined;
if (modelDisplay && provider) return `${modelDisplay} · ${provider}`;
return modelDisplay || provider || "Unknown model";
}
function getPrompt(job: Job, promptsMap?: Map<string, Prompts>): string {
const promptToken = job.request.maybe_prompt_token;
const cached = promptToken
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
: undefined;
return (
cached?.maybe_positive_prompt || job.request.maybe_raw_inference_text || ""
);
}
function getModelId(job: Job, promptsMap?: Map<string, Prompts>): string {
const promptToken = job.request.maybe_prompt_token;
const cachedPrompt = promptToken
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
: undefined;
return cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
}
function jobToInProgress(
job: Job,
promptsMap: Map<string, Prompts>,
): InProgressJob {
const now = Date.now();
const createdMs = new Date(job.created_at).getTime();
const modelType = job.request.maybe_model_type;
const mediaClass = getJobMediaClass(job);
let duration = taskDurationCache.get(job.job_token);
if (!duration) {
const model = modelType
? ALL_MODELS_LIST.find(
(m) => m.tauriId === modelType || m.id === modelType,
)
: undefined;
duration = model?.progressBarTime ?? DEFAULT_DURATION_MS[mediaClass];
taskDurationCache.set(job.job_token, duration);
}
const elapsed = now - createdMs;
const progress = Math.min(95, (elapsed / duration) * 100);
const estimatedTimeLeftMs = Math.max(0, duration - elapsed);
return {
id: job.job_token,
prompt: getPrompt(job, promptsMap),
modelId: getModelId(job, promptsMap),
modelLabel: getModelLabel(job, promptsMap),
progress,
estimatedTimeLeftMs,
createdAt: job.created_at,
promptToken: job.request.maybe_prompt_token ?? undefined,
mediaClass,
};
}
function jobToFailed(job: Job, promptsMap: Map<string, Prompts>): FailedJob {
const failureCategory =
job.status.maybe_failure_category_updated ||
job.status.maybe_failure_category;
const rawMessage =
job.status.maybe_failure_message ||
job.status.maybe_extra_status_description;
const failureReason = failureCategory
? FAILURE_REASON_LABEL[failureCategory] || rawMessage || undefined
: rawMessage || undefined;
const failureMessage =
rawMessage && failureCategory !== "unknown" ? rawMessage : undefined;
const promptToken = job.request.maybe_prompt_token ?? undefined;
const promptData = promptToken ? promptsMap.get(promptToken) : undefined;
const refImageUrl = pickFirstRefImageUrl(promptData);
return {
id: job.job_token,
prompt: getPrompt(job, promptsMap),
modelId: getModelId(job, promptsMap),
modelLabel: getModelLabel(job, promptsMap),
failureReason,
failureMessage,
status: job.status.status,
createdAt: job.created_at,
promptToken,
refImageUrl,
mediaClass: getJobMediaClass(job),
};
}
// Semantics that aren't still images — videos and audio clips can't be rendered
// as the faded backdrop behind the error state, so skip them when picking the
// reference to display.
const NON_IMAGE_REF_SEMANTICS = new Set(["vid_ref", "audioref"]);
function pickFirstRefImageUrl(
promptData: Prompts | undefined,
): string | undefined {
const refs = promptData?.maybe_context_images;
if (!refs?.length) return undefined;
for (const ref of refs) {
if (NON_IMAGE_REF_SEMANTICS.has(ref.semantic)) continue;
const url = ref.media_links?.cdn_url;
if (url) return url;
}
return undefined;
}
function jobToGalleryItem(
job: Job,
promptsMap?: Map<string, Prompts>,
): GalleryItem | null {
const result = job.maybe_result;
if (!result?.entity_token) return null;
const mediaClass = getJobMediaClass(job);
// Audio has no image thumbnail — its card renders a waveform player.
const thumbnail =
mediaClass === "audio"
? null
: getMediaThumbnail(result.media_links, mediaClass, {
size: THUMBNAIL_SIZES.LARGE,
});
return {
id: result.entity_token,
label: getPrompt(job, promptsMap) || "Generation",
thumbnail,
fullImage: result.media_links?.cdn_url || null,
// Sort by job creation time (not completion time) so the completed card
// occupies the same Masonry slot the pending card held — no layout shift.
createdAt: job.created_at,
mediaClass,
modelId: job.request.maybe_model_type ?? undefined,
};
}
/** Expand a single GalleryItem into its batch siblings (if any). */
async function expandBatchItems(
item: GalleryItem,
mediaFilesApi: MediaFilesApi,
): Promise<GalleryItem[]> {
try {
const mediaResponse = await mediaFilesApi.GetMediaFileByToken({
mediaFileToken: item.id,
});
const batchToken = (mediaResponse.data as any)?.maybe_batch_token;
if (!batchToken) return [item];
const batchResponse = await mediaFilesApi.GetMediaFilesByBatchToken({
batchToken,
});
if (!batchResponse.success || !batchResponse.data?.length) return [item];
return batchResponse.data
.map((file: any): GalleryItem | null => {
const cdnUrl = file.media_links?.cdn_url;
if (!cdnUrl) return null;
const thumbnail =
item.mediaClass === "audio"
? null
: getMediaThumbnail(file.media_links, item.mediaClass, {
size: THUMBNAIL_SIZES.LARGE,
}) || cdnUrl;
return {
id: file.token,
label: item.label,
thumbnail,
fullImage: cdnUrl,
createdAt: item.createdAt,
mediaClass: item.mediaClass,
modelId: item.modelId,
batchImageToken: batchToken,
durationMillis: file.maybe_duration_millis ?? undefined,
};
})
.filter((i): i is GalleryItem => i !== null);
} catch {
return [item];
}
}
// ── Hook ───────────────────────────────────────────────────────────────────
export function useGenerationJobs(options: {
mediaType: GenerationMediaClass;
enabled?: boolean;
}) {
const { mediaType, enabled = true } = options;
const apiRef = useRef(new JobsApi());
const mediaApiRef = useRef(new MediaFilesApi());
const [inProgressJobs, setInProgressJobs] = useState<Job[]>([]);
const [failedJobsRaw, setFailedJobsRaw] = useState<Job[]>([]);
const [newlyCompleted, setNewlyCompleted] = useState<GalleryItem[]>([]);
const prevCompletedIdsRef = useRef<Set<string>>(new Set());
const initialLoadDoneRef = useRef(false);
const promptTokens = useMemo(() => {
const tokens: string[] = [];
for (const j of inProgressJobs) {
if (j.request.maybe_prompt_token)
tokens.push(j.request.maybe_prompt_token);
}
for (const j of failedJobsRaw) {
if (j.request.maybe_prompt_token)
tokens.push(j.request.maybe_prompt_token);
}
return tokens;
}, [inProgressJobs, failedJobsRaw]);
const promptsMap = usePrompts(promptTokens);
const inProgress = useMemo(
() =>
inProgressJobs
.slice()
.sort(
(a, b) =>
new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
)
.map((j) => jobToInProgress(j, promptsMap)),
[inProgressJobs, promptsMap],
);
const failed = useMemo(
() =>
failedJobsRaw
.slice()
.sort(
(a, b) =>
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
)
.map((j) => jobToFailed(j, promptsMap)),
[failedJobsRaw, promptsMap],
);
const load = useCallback(async () => {
try {
const response = await apiRef.current.ListRecentJobs();
if (!response.success || !response.data) return;
const jobs: Job[] = response.data;
// Filter by media type
const filtered = jobs.filter((j) => getJobMediaType(j) === mediaType);
const newInProgress = filtered.filter((j) =>
IN_PROGRESS_STATUSES.has(j.status.status),
);
const newFailed = filtered.filter((j) =>
FAILED_STATUSES.has(j.status.status),
);
// Completed
const completedJobs = filtered.filter((j) =>
COMPLETED_STATUSES.has(j.status.status),
);
const completedIdSet = new Set(completedJobs.map((j) => j.job_token));
// Detect newly completed (skip on first load to avoid flooding)
let expandedNewItems: GalleryItem[] = [];
if (initialLoadDoneRef.current) {
const newOnes = completedJobs.filter(
(j) => !prevCompletedIdsRef.current.has(j.job_token),
);
if (newOnes.length > 0) {
const items = newOnes
.map((j) => jobToGalleryItem(j))
.filter((item): item is GalleryItem => item !== null);
if (items.length > 0) {
// Await expansion so the pending card and its completed replacement
// commit in the same React render — no "remove then add" gap.
const expanded = await Promise.all(
items.map((item) => expandBatchItems(item, mediaApiRef.current)),
);
expandedNewItems = expanded.flat();
}
}
}
initialLoadDoneRef.current = true;
prevCompletedIdsRef.current = completedIdSet;
// Prune duration cache
const activeIds = new Set(newInProgress.map((j) => j.job_token));
for (const id of taskDurationCache.keys()) {
if (!activeIds.has(id)) taskDurationCache.delete(id);
}
if (expandedNewItems.length > 0) {
setNewlyCompleted((prev) => {
const existingIds = new Set(prev.map((i) => i.id));
const fresh = expandedNewItems.filter((i) => !existingIds.has(i.id));
return [...fresh, ...prev];
});
}
setInProgressJobs(newInProgress);
setFailedJobsRaw(newFailed);
} catch {
// ignore
}
}, [mediaType]);
// Poll every 5 seconds + listen for task-queue-update events.
// Skip entirely when disabled (e.g. user is logged out) — otherwise we'd
// hit an authenticated endpoint every 5s for nothing, which on mobile
// Safari shows up as periodic main-thread jank during the menu animation.
useEffect(() => {
if (!enabled) return;
load();
const intervalId = setInterval(load, 5000);
const handleTaskUpdate = () => load();
window.addEventListener("task-queue-update", handleTaskUpdate);
return () => {
clearInterval(intervalId);
window.removeEventListener("task-queue-update", handleTaskUpdate);
};
}, [load, enabled]);
const dismissFailed = useCallback(async (jobToken: string) => {
try {
await apiRef.current.DeleteJobByToken(jobToken);
setFailedJobsRaw((prev) => prev.filter((f) => f.job_token !== jobToken));
} catch {
// ignore
}
}, []);
return {
inProgress,
failed,
newlyCompleted,
dismissFailed,
refresh: load,
};
}
@@ -8,6 +8,9 @@
{
"path": "../loading-spinner"
},
{
"path": "../audio-player"
},
{
"path": "../../common"
},
@@ -49,6 +49,9 @@
{
"path": "../loading-spinner/tsconfig.lib.json"
},
{
"path": "../audio-player/tsconfig.lib.json"
},
{
"path": "../../common/tsconfig.lib.json"
},
@@ -10,6 +10,7 @@ import {
faDownToLine,
faGlobe,
faMagnifyingGlass,
faMusic,
faPause,
faPencil,
faPlay,
@@ -21,6 +22,7 @@ import {
import { MediaFileDelete } from "@storyteller/tauri-api";
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
import { Viewer3D } from "@storyteller/ui-viewer-3d";
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
import {
useEffect,
useState,
@@ -608,6 +610,19 @@ export function LightboxModal({
<source src={selectedImageUrl as string} type="video/mp4" />
Your browser does not support the video tag.
</video>
) : mediaClass === "audio" ? (
<div className="flex h-full w-full items-center justify-center px-4 sm:px-8">
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-white/[0.04] px-2 py-8">
<FontAwesomeIcon
icon={faMusic}
className="mx-auto mb-5 block text-3xl text-white/20"
/>
<WaveformAudioPlayer
key={selectedImageUrl as string}
src={selectedImageUrl as string}
/>
</div>
</div>
) : (
<div className="flex h-full w-full flex-col justify-center">
<div
@@ -692,7 +707,8 @@ export function LightboxModal({
{!mediaLoaded &&
selectedImageUrl &&
mediaClass !== "dimensional" && (
mediaClass !== "dimensional" &&
mediaClass !== "audio" && (
<div className="absolute inset-0 bg-ui-panel flex items-center justify-center">
<LoadingSpinner className="h-12 w-12 text-base-fg" />
</div>
@@ -29,6 +29,9 @@
{
"path": "../loading-spinner"
},
{
"path": "../audio-player"
},
{
"path": "../../tauri-api"
},
@@ -70,6 +70,9 @@
{
"path": "../loading-spinner/tsconfig.lib.json"
},
{
"path": "../audio-player/tsconfig.lib.json"
},
{
"path": "../../tauri-api/tsconfig.lib.json"
},
@@ -3,8 +3,12 @@ export * from "./lib/PromptBox3D";
export * from "./lib/PromptBoxEdit";
export * from "./lib/PromptBoxVideo";
export * from "./lib/PromptBoxImage";
export * from "./lib/PromptBoxAudio";
export * from "./lib/useAutoGrowEditorHeight";
export * from "./lib/PromptFullscreenModal";
export * from "./lib/PromptFullscreenButton";
export * from "./lib/promptStore";
export * from "./lib/PromptBoxErrorBoundary";
export * from "./lib/common/AudioTuningPopover";
export * from "./lib/common/SoundsSettingsPopover";
export * from "./lib/common/StylePromptRow";
@@ -0,0 +1,498 @@
import { useMemo, useRef, useState, ReactNode } from "react";
import { toast } from "@storyteller/ui-toaster";
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { GenerateButton, ToggleButton } from "@storyteller/ui-button";
import {
faChevronDown,
faChevronUp,
faMicrophoneLines,
faMicrophoneSlash,
faRepeat,
} from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import type { OmniGenAudioModelDetails, UploadMediaFn } from "@storyteller/api";
import {
enqueueAudioGeneration,
AUDIO_MODELS_REQUIRING_AUDIO_REF,
OMNI_GENERATE_OUTAGE_MESSAGE,
type AudioGenerationSettings,
} from "@storyteller/omni-gen";
import {
getCreatorIconPathForModelId,
getModelDescription,
getModelInfo,
} from "@storyteller/model-list";
import { usePromptAudioStore, useEnterToGenerateStore } from "./promptStore";
import { useAutoGrowEditorHeight } from "./useAutoGrowEditorHeight";
import {
PromptFullscreenModal,
useFullscreenPrompt,
} from "./PromptFullscreenModal";
import { PromptFullscreenButton } from "./PromptFullscreenButton";
import { gtagEvent } from "@storyteller/google-analytics";
import { twMerge } from "tailwind-merge";
import { AudioReferenceRow } from "./common/AudioReferenceRow";
import { AudioTuningPopover } from "./common/AudioTuningPopover";
import { SoundsSettingsPopover } from "./common/SoundsSettingsPopover";
import { StylePromptRow } from "./common/StylePromptRow";
const DEFAULT_AUDIO_MODEL_ID = "suno_music";
const AUDIO_REF_MAX_DURATION_SECONDS = 600;
// Capability flags are serde-skipped when absent — only `true` counts.
const supports = (flag: boolean | null | undefined): boolean => flag === true;
interface PromptBoxAudioProps {
// Audio models from GET /v1/omni_gen/models/audio (useOmniGenAudioModels).
models: OmniGenAudioModelDetails[];
uploadAudio?: UploadMediaFn;
uploadImage?: UploadMediaFn;
// Fired after a successful enqueue with every created job token (one
// request can create several Suno clips).
onEnqueuePressed?: (jobTokens: string[]) => void | Promise<void>;
credits?: number | null;
}
export const PromptBoxAudio = ({
models,
uploadAudio,
uploadImage,
onEnqueuePressed,
credits,
}: PromptBoxAudioProps) => {
const prompt = usePromptAudioStore((s) => s.prompt);
const setPrompt = usePromptAudioStore((s) => s.setPrompt);
const stylePrompt = usePromptAudioStore((s) => s.stylePrompt);
const setStylePrompt = usePromptAudioStore((s) => s.setStylePrompt);
const selectedModelId = usePromptAudioStore((s) => s.selectedModelId);
const setSelectedModelId = usePromptAudioStore((s) => s.setSelectedModelId);
const isInstrumental = usePromptAudioStore((s) => s.isInstrumental);
const setIsInstrumental = usePromptAudioStore((s) => s.setIsInstrumental);
const keepLyrics = usePromptAudioStore((s) => s.keepLyrics);
const setKeepLyrics = usePromptAudioStore((s) => s.setKeepLyrics);
const isLoopable = usePromptAudioStore((s) => s.isLoopable);
const setIsLoopable = usePromptAudioStore((s) => s.setIsLoopable);
const bpm = usePromptAudioStore((s) => s.bpm);
const setBpm = usePromptAudioStore((s) => s.setBpm);
const musicalKey = usePromptAudioStore((s) => s.musicalKey);
const setMusicalKey = usePromptAudioStore((s) => s.setMusicalKey);
const sampleRateHz = usePromptAudioStore((s) => s.sampleRateHz);
const setSampleRateHz = usePromptAudioStore((s) => s.setSampleRateHz);
const speed = usePromptAudioStore((s) => s.speed);
const setSpeed = usePromptAudioStore((s) => s.setSpeed);
const volume = usePromptAudioStore((s) => s.volume);
const setVolume = usePromptAudioStore((s) => s.setVolume);
const pitch = usePromptAudioStore((s) => s.pitch);
const setPitch = usePromptAudioStore((s) => s.setPitch);
const referenceAudios = usePromptAudioStore((s) => s.referenceAudios);
const setReferenceAudios = usePromptAudioStore((s) => s.setReferenceAudios);
const referenceImages = usePromptAudioStore((s) => s.referenceImages);
const setReferenceImages = usePromptAudioStore((s) => s.setReferenceImages);
const [isEnqueueing, setIsEnqueueing] = useState(false);
const [isFocused, setIsFocused] = useState(false);
const enterToGenerate = useEnterToGenerateStore((s) => s.enabled);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const { isExpanded, toggleExpand } = useAutoGrowEditorHeight(
textareaRef,
prompt,
);
const { isFullscreen, openFullscreen, closeFullscreen } =
useFullscreenPrompt();
const selectedModel = useMemo((): OmniGenAudioModelDetails | undefined => {
if (!models.length) return undefined;
if (selectedModelId) {
const chosen = models.find((m) => m.model === selectedModelId);
if (chosen) return chosen;
}
return (
models.find((m) => m.model === DEFAULT_AUDIO_MODEL_ID) ?? models[0]
);
}, [models, selectedModelId]);
// Capability gating
const styleSupported = supports(selectedModel?.style_prompt_supported);
const hasInstrumental = supports(
selectedModel?.instrumental_toggle_supported,
);
const hasKeepLyrics = supports(selectedModel?.keep_lyrics_supported);
const hasLoopable = supports(selectedModel?.loopable_toggle_supported);
const hasBpm = supports(selectedModel?.bpm_supported);
const hasMusicalKey = supports(selectedModel?.musical_key_supported);
const sampleRateOptions = selectedModel?.sample_rate_hz_options ?? null;
const hasSpeed = supports(selectedModel?.speed_supported);
const hasVolume = supports(selectedModel?.volume_supported);
const hasPitch = supports(selectedModel?.pitch_supported);
const hasTuning =
!!sampleRateOptions?.length || hasSpeed || hasVolume || hasPitch;
const audioRefsSupported = supports(
selectedModel?.audio_references_supported,
);
const maxAudioRefs = selectedModel?.audio_references_max ?? 1;
const imageRefsSupported = supports(
selectedModel?.image_references_supported,
);
const requiresAudioRef = AUDIO_MODELS_REQUIRING_AUDIO_REF.has(
selectedModel?.model ?? "",
);
const missingRequiredAudioRef =
requiresAudioRef && referenceAudios.length !== 1;
const effectiveSampleRate =
sampleRateHz != null && sampleRateOptions?.includes(sampleRateHz)
? sampleRateHz
: (selectedModel?.sample_rate_hz_default ??
sampleRateOptions?.[0] ??
null);
const modelItems = useMemo(
(): PopoverItem[] =>
models.map((model) => ({
label: model.full_name || model.model,
selected: model.model === selectedModel?.model,
description: getModelDescription(model.model, model.extra_info_short),
info: getModelInfo(model.model, model.extra_info) || undefined,
icon: (
<img
src={getCreatorIconPathForModelId(model.model)}
alt={`${model.model} logo`}
className="h-4 w-4 icon-auto-contrast"
/>
),
action: model.model,
})),
[models, selectedModel?.model],
);
// Seed Audio can't combine audio and image references — adding one kind
// clears the other so the request is always valid.
const handleReferenceAudiosChange = (audios: typeof referenceAudios) => {
if (audios.length > 0 && referenceImages.length > 0) {
setReferenceImages([]);
toast.error("Removed image reference — it can't be combined with audio");
}
setReferenceAudios(audios);
};
const handleReferenceImagesChange = (images: typeof referenceImages) => {
if (images.length > 0 && referenceAudios.length > 0) {
setReferenceAudios([]);
toast.error("Removed audio reference — it can't be combined with an image");
}
setReferenceImages(images);
};
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
setPrompt(e.target.value);
};
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
e.preventDefault();
const pastedText = e.clipboardData.getData("text");
const target = e.currentTarget;
const { selectionStart, selectionEnd, value } = target;
const next =
value.slice(0, selectionStart) + pastedText + value.slice(selectionEnd);
setPrompt(next);
requestAnimationFrame(() => {
const pos = Math.min(selectionStart + pastedText.length, next.length);
textareaRef.current?.setSelectionRange(pos, pos);
});
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key !== "Enter") return;
if (enterToGenerate && !e.shiftKey) {
e.preventDefault();
handleEnqueue();
}
};
const handleEnqueue = async () => {
if (!prompt.trim() || !selectedModel || isEnqueueing) return;
if (missingRequiredAudioRef) {
toast.error(
`${selectedModel.full_name ?? "This model"} needs an audio track to work from — add one first`,
);
return;
}
setIsEnqueueing(true);
gtagEvent("enqueue_audio");
try {
const settings: AudioGenerationSettings = {
prompt,
stylePrompt,
audioMediaTokens: referenceAudios
.map((audio) => audio.mediaToken)
.filter((token) => token.length > 0),
imageMediaTokens: referenceImages
.map((image) => image.mediaToken)
.filter((token) => token.length > 0),
isInstrumental,
keepLyrics,
isLoopable,
bpm,
musicalKey,
sampleRateHz: effectiveSampleRate,
speed,
volume,
pitch,
};
const result = await enqueueAudioGeneration(selectedModel, settings);
if (!result.success) {
if (result.errorCode === 402) {
toast.error("Not enough credits for this generation");
} else if (result.errorCode != null && result.errorCode >= 500) {
toast.error(OMNI_GENERATE_OUTAGE_MESSAGE);
} else {
toast.error(result.error ?? "Failed to start audio generation");
}
return;
}
await onEnqueuePressed?.(result.jobTokens);
} catch (err) {
console.error("PromptBoxAudio - enqueue failed", err);
toast.error("Failed to start audio generation. Please try again.");
} finally {
setIsEnqueueing(false);
}
};
const modelSelector: ReactNode = (
<Tooltip content="Model" position="top" className="z-50" closeOnClick>
<PopoverMenu
items={modelItems}
onSelect={(item) => {
if (item.action) setSelectedModelId(item.action);
}}
mode="toggle"
panelTitle="Select Model"
panelClassName="w-[360px]"
richList
triggerIcon={
<img
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
alt=""
className="h-4 w-4 icon-auto-contrast"
/>
}
triggerLabel={selectedModel?.full_name ?? "Model"}
/>
</Tooltip>
);
const toggleButtons: ReactNode = (
<>
{hasInstrumental && (
<Tooltip
content={isInstrumental ? "Instrumental: ON" : "Instrumental: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={isInstrumental}
icon={faMicrophoneSlash}
activeIcon={faMicrophoneSlash}
label="Instrumental"
onClick={() => setIsInstrumental(!isInstrumental)}
/>
</Tooltip>
)}
{hasKeepLyrics && (
<Tooltip
content={keepLyrics ? "Keep lyrics: ON" : "Keep lyrics: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={keepLyrics}
icon={faMicrophoneLines}
activeIcon={faMicrophoneLines}
label="Keep lyrics"
onClick={() => setKeepLyrics(!keepLyrics)}
/>
</Tooltip>
)}
{hasLoopable && (
<Tooltip
content={isLoopable ? "Loop: ON" : "Loop: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={isLoopable}
icon={faRepeat}
activeIcon={faRepeat}
label="Loop"
onClick={() => setIsLoopable(!isLoopable)}
/>
</Tooltip>
)}
</>
);
const settingsPopovers: ReactNode = (
<>
{(hasBpm || hasMusicalKey) && (
<SoundsSettingsPopover
showBpm={hasBpm}
bpm={bpm}
onBpmChange={setBpm}
showMusicalKey={hasMusicalKey}
musicalKey={musicalKey}
onMusicalKeyChange={setMusicalKey}
/>
)}
{hasTuning && (
<AudioTuningPopover
sampleRateOptions={sampleRateOptions}
sampleRateHz={effectiveSampleRate}
onSampleRateChange={setSampleRateHz}
showSpeed={hasSpeed}
speed={speed}
onSpeedChange={setSpeed}
showVolume={hasVolume}
volume={volume}
onVolumeChange={setVolume}
showPitch={hasPitch}
pitch={pitch}
onPitchChange={setPitch}
/>
)}
</>
);
const referenceRow: ReactNode =
audioRefsSupported || imageRefsSupported ? (
<AudioReferenceRow
referenceAudios={referenceAudios}
onReferenceAudiosChange={handleReferenceAudiosChange}
maxAudioCount={audioRefsSupported ? maxAudioRefs : 0}
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
uploadAudio={uploadAudio}
audioRequired={requiresAudioRef}
imageSupported={imageRefsSupported}
referenceImages={referenceImages}
onReferenceImagesChange={handleReferenceImagesChange}
uploadImage={uploadImage}
/>
) : null;
return (
<>
<div className="relative z-20 flex flex-col">
<div
className={twMerge(
"glass relative w-full rounded-2xl p-4",
isFocused
? "ring-1 ring-primary border-primary"
: "ring-1 ring-transparent",
)}
>
{referenceRow}
<div className="flex justify-center gap-2">
<div className="promptbox-resize-wrap relative flex-1">
<textarea
ref={textareaRef}
rows={1}
placeholder="Describe the music or sound you want..."
className="promptbox-scrollbar text-md mb-2 min-h-[2.5em] w-full resize-y overflow-y-auto rounded bg-transparent pb-2 pr-8 pt-1 text-base-fg placeholder-base-fg/60 transition-[height] duration-200 ease-out focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
/>
<PromptFullscreenButton onClick={openFullscreen} />
</div>
</div>
{styleSupported && (
<StylePromptRow value={stylePrompt} onChange={setStylePrompt} />
)}
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex flex-wrap items-center gap-2">
{modelSelector}
{toggleButtons}
{settingsPopovers}
</div>
<div className="flex items-center gap-2">
{missingRequiredAudioRef && (
<span className="flex animate-pulse items-center gap-1.5 text-xs font-medium text-red-500">
Audio track required
</span>
)}
<GenerateButton
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={undefined}
onClick={handleEnqueue}
disabled={!prompt.trim() || missingRequiredAudioRef}
loading={isEnqueueing}
credits={credits}
>
Generate
</GenerateButton>
</div>
</div>
<div className="absolute -bottom-1 left-1/2 -translate-x-1/2">
<Tooltip
content={isExpanded ? "Collapse" : "Expand"}
position="top"
className="-mb-2"
>
<button
type="button"
onClick={toggleExpand}
className="text-base-fg/30 hover:text-base-fg/90 transition-colors px-3 py-0.5"
>
<FontAwesomeIcon
icon={isExpanded ? faChevronUp : faChevronDown}
className="text-xs"
/>
</button>
</Tooltip>
</div>
</div>
</div>
<PromptFullscreenModal
isOpen={isFullscreen}
onClose={closeFullscreen}
promptLength={prompt.length}
maxLength={Infinity}
footerControls={
<>
{modelSelector}
{toggleButtons}
{settingsPopovers}
</>
}
>
<textarea
placeholder="Describe the music or sound you want..."
className="promptbox-scrollbar text-md h-full min-h-0 w-full resize-none overflow-y-auto rounded bg-transparent text-base-fg placeholder-base-fg/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
/>
</PromptFullscreenModal>
</>
);
};
@@ -0,0 +1,329 @@
import { useCallback, useRef, useState } from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faImage,
faMusic,
faPlay,
faPlus,
faStop,
faXmark,
} from "@fortawesome/pro-solid-svg-icons";
import { toast } from "@storyteller/ui-toaster";
import { UploaderStates } from "@storyteller/common";
import type { UploadMediaFn } from "@storyteller/api";
import type { RefAudio, RefImage } from "../promptStore";
export interface AudioReferenceRowProps {
referenceAudios: RefAudio[];
onReferenceAudiosChange: (audios: RefAudio[]) => void;
maxAudioCount: number;
maxAudioRefDuration: number;
uploadAudio?: UploadMediaFn;
// Whether the audio reference is required (remix/sample source).
audioRequired?: boolean;
// Optional single-image reference section (Seed Audio).
imageSupported?: boolean;
referenceImages?: RefImage[];
onReferenceImagesChange?: (images: RefImage[]) => void;
uploadImage?: UploadMediaFn;
className?: string;
}
// Reference row for the audio promptbox: upload an audio track (the
// remix/sample source or Seed Audio refs), plus an optional image reference
// for models that support one. Rendered inside the glass card, above the
// prompt textarea.
export function AudioReferenceRow({
referenceAudios,
onReferenceAudiosChange,
maxAudioCount,
maxAudioRefDuration,
uploadAudio,
audioRequired = false,
imageSupported = false,
referenceImages = [],
onReferenceImagesChange,
uploadImage,
className = "",
}: AudioReferenceRowProps) {
const audioInputRef = useRef<HTMLInputElement>(null);
const imageInputRef = useRef<HTMLInputElement>(null);
const [isUploadingAudio, setIsUploadingAudio] = useState(false);
const [isUploadingImage, setIsUploadingImage] = useState(false);
// Latest-value refs so async upload callbacks append to fresh state.
const referenceAudiosRef = useRef(referenceAudios);
referenceAudiosRef.current = referenceAudios;
const referenceImagesRef = useRef(referenceImages);
referenceImagesRef.current = referenceImages;
const handleAudioFileUpload = async (
event: React.ChangeEvent<HTMLInputElement>,
) => {
const files = Array.from(event.target.files || []);
if (files.length === 0) return;
const availableSlots = Math.max(0, maxAudioCount - referenceAudios.length);
const filesToProcess = files.slice(0, availableSlots);
for (const file of filesToProcess) {
const duration = await getAudioFileDuration(file);
const currentTotal = referenceAudiosRef.current.reduce(
(sum, audio) => sum + audio.duration,
0,
);
if (currentTotal + duration > maxAudioRefDuration) {
toast.error(
`Total audio duration cannot exceed ${maxAudioRefDuration}s`,
);
break;
}
if (uploadAudio) {
setIsUploadingAudio(true);
await uploadAudio({
title: `reference-audio-${Math.random().toString(36).substring(2, 15)}`,
assetFile: file,
progressCallback: (newState) => {
if (newState.status === UploaderStates.success && newState.data) {
const refAudio: RefAudio = {
id: Math.random().toString(36).substring(7),
url: URL.createObjectURL(file),
file,
mediaToken: newState.data,
duration,
};
setIsUploadingAudio(false);
onReferenceAudiosChange([
...referenceAudiosRef.current,
refAudio,
]);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
setIsUploadingAudio(false);
toast.error("Audio upload failed. Please try again.");
}
},
});
}
}
if (audioInputRef.current) audioInputRef.current.value = "";
};
const handleImageFileUpload = async (
event: React.ChangeEvent<HTMLInputElement>,
) => {
const file = (event.target.files || [])[0];
if (!file || !uploadImage) return;
setIsUploadingImage(true);
await uploadImage({
title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
assetFile: file,
progressCallback: (newState) => {
if (newState.status === UploaderStates.success && newState.data) {
const refImage: RefImage = {
id: Math.random().toString(36).substring(7),
url: URL.createObjectURL(file),
file,
mediaToken: newState.data,
};
setIsUploadingImage(false);
onReferenceImagesChange?.([refImage]);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
setIsUploadingImage(false);
toast.error("Image upload failed. Please try again.");
}
},
});
if (imageInputRef.current) imageInputRef.current.value = "";
};
const removeAudio = (id: string) => {
onReferenceAudiosChange(referenceAudios.filter((a) => a.id !== id));
if (audioInputRef.current) audioInputRef.current.value = "";
};
return (
<div className={`flex flex-col gap-2 pb-2.5 ${className}`}>
<input
ref={audioInputRef}
type="file"
accept="audio/*"
className="hidden"
multiple={maxAudioCount > 1}
onChange={handleAudioFileUpload}
/>
{imageSupported && (
<input
ref={imageInputRef}
type="file"
accept="image/*"
className="hidden"
onChange={handleImageFileUpload}
/>
)}
<div className="flex flex-wrap items-center gap-2">
<div className="flex items-center gap-2 text-base-fg opacity-90">
<FontAwesomeIcon icon={faMusic} className="h-3.5 w-3.5" />
<span className="text-sm font-medium">
Audio Track{" "}
<span className="font-semibold text-base-fg/60">
({referenceAudios.length}/{maxAudioCount})
</span>
{audioRequired && referenceAudios.length === 0 && (
<span className="ml-1.5 text-xs font-medium text-red-500">
required
</span>
)}
</span>
</div>
{referenceAudios.map((audio, index) => (
<AudioRefTile
key={audio.id}
audio={audio}
index={index}
onRemove={removeAudio}
/>
))}
{referenceAudios.length < maxAudioCount && (
<button
type="button"
onClick={() => audioInputRef.current?.click()}
disabled={isUploadingAudio}
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
>
<FontAwesomeIcon icon={faPlus} className="h-3 w-3" />
{isUploadingAudio ? "Uploading…" : "Add audio"}
</button>
)}
{imageSupported && (
<>
<div className="ms-2 flex items-center gap-2 text-base-fg opacity-90">
<FontAwesomeIcon icon={faImage} className="h-3.5 w-3.5" />
<span className="text-sm font-medium">Image</span>
</div>
{referenceImages.map((image) => (
<div
key={image.id}
className="group relative h-9 w-9 overflow-hidden rounded-lg border border-ui-controls-border"
>
<img
src={image.url}
alt="Reference"
className="h-full w-full object-cover"
/>
<button
type="button"
aria-label="Remove image"
onClick={() => onReferenceImagesChange?.([])}
className="absolute inset-0 flex items-center justify-center bg-black/60 opacity-0 transition-opacity group-hover:opacity-100"
>
<FontAwesomeIcon
icon={faXmark}
className="h-3 w-3 text-white"
/>
</button>
</div>
))}
{referenceImages.length === 0 && (
<button
type="button"
onClick={() => imageInputRef.current?.click()}
disabled={isUploadingImage}
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
>
<FontAwesomeIcon icon={faPlus} className="h-3 w-3" />
{isUploadingImage ? "Uploading…" : "Add image"}
</button>
)}
</>
)}
</div>
</div>
);
}
function AudioRefTile({
audio,
index,
onRemove,
}: {
audio: RefAudio;
index: number;
onRemove: (id: string) => void;
}) {
const audioRef = useRef<HTMLAudioElement | null>(null);
const [isPlaying, setIsPlaying] = useState(false);
const togglePlay = useCallback(() => {
if (isPlaying) {
audioRef.current?.pause();
audioRef.current = null;
setIsPlaying(false);
return;
}
const element = new Audio(audio.url);
element.onended = () => {
audioRef.current = null;
setIsPlaying(false);
};
audioRef.current = element;
void element.play().catch(() => setIsPlaying(false));
setIsPlaying(true);
}, [audio.url, isPlaying]);
return (
<div className="group flex h-9 items-center gap-2 rounded-lg border border-ui-controls-border bg-ui-controls px-2.5">
<button
type="button"
aria-label={isPlaying ? "Stop" : "Play"}
onClick={togglePlay}
className="flex h-5 w-5 items-center justify-center rounded-full bg-white/90 text-black transition-transform hover:scale-105"
>
<FontAwesomeIcon
icon={isPlaying ? faStop : faPlay}
className={`h-2 w-2 ${isPlaying ? "" : "ml-px"}`}
/>
</button>
<span className="text-xs font-medium text-base-fg/80">
Audio {index + 1} · {audio.duration}s
</span>
<button
type="button"
aria-label="Remove audio"
onClick={() => onRemove(audio.id)}
className="flex h-4 w-4 items-center justify-center rounded text-base-fg/40 transition-colors hover:text-base-fg"
>
<FontAwesomeIcon icon={faXmark} className="h-2.5 w-2.5" />
</button>
</div>
);
}
function getAudioFileDuration(file: File): Promise<number> {
return new Promise((resolve) => {
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.onloadedmetadata = () => {
URL.revokeObjectURL(audio.src);
resolve(Math.round(audio.duration));
};
audio.onerror = () => {
URL.revokeObjectURL(audio.src);
resolve(0);
};
audio.src = URL.createObjectURL(file);
});
}
@@ -0,0 +1,166 @@
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faSliders } from "@fortawesome/pro-solid-svg-icons";
import { PopoverMenu } from "@storyteller/ui-popover";
import { SliderV2 } from "@storyteller/ui-sliderv2";
// Output-shaping controls for models that support them (Seed Audio):
// sample rate, playback speed, volume, and pitch. Ranges mirror the
// backend's validate_audio_request limits.
export const AUDIO_SPEED_MIN = 0.5;
export const AUDIO_SPEED_MAX = 2.0;
export const AUDIO_VOLUME_MIN = 0.5;
export const AUDIO_VOLUME_MAX = 2.0;
export const AUDIO_PITCH_MIN = -12;
export const AUDIO_PITCH_MAX = 12;
export interface AudioTuningPopoverProps {
// Sample rate section renders only when options are provided.
sampleRateOptions?: number[] | null;
sampleRateHz?: number | null;
onSampleRateChange?: (hz: number) => void;
showSpeed?: boolean;
speed?: number;
onSpeedChange?: (value: number) => void;
showVolume?: boolean;
volume?: number;
onVolumeChange?: (value: number) => void;
showPitch?: boolean;
pitch?: number;
onPitchChange?: (value: number) => void;
}
export function AudioTuningPopover({
sampleRateOptions,
sampleRateHz,
onSampleRateChange,
showSpeed = false,
speed = 1,
onSpeedChange,
showVolume = false,
volume = 1,
onVolumeChange,
showPitch = false,
pitch = 0,
onPitchChange,
}: AudioTuningPopoverProps) {
const showSampleRate =
!!sampleRateOptions?.length && onSampleRateChange !== undefined;
return (
<PopoverMenu
mode="default"
panelTitle="Tuning"
triggerIcon={<FontAwesomeIcon icon={faSliders} className="h-3.5 w-3.5" />}
triggerLabel="Tuning"
>
<div className="w-60 space-y-3.5 pb-0.5">
{showSampleRate && (
<div>
<div className="mb-1.5 text-xs font-medium text-base-fg/60">
Sample rate
</div>
<div className="flex flex-wrap gap-1">
{sampleRateOptions.map((hz) => (
<button
key={hz}
type="button"
onClick={() => onSampleRateChange?.(hz)}
className={`rounded-md border px-2 py-1 text-xs font-medium transition-colors ${
hz === sampleRateHz
? "border-white/30 bg-white/15 text-base-fg"
: "border-transparent bg-white/5 text-base-fg/60 hover:bg-white/10 hover:text-base-fg"
}`}
>
{formatSampleRateHz(hz)}
</button>
))}
</div>
</div>
)}
{showSpeed && (
<TuningSliderRow
label="Speed"
valueLabel={`${speed.toFixed(2)}×`}
min={AUDIO_SPEED_MIN}
max={AUDIO_SPEED_MAX}
step={0.05}
value={speed}
onChange={(value) => onSpeedChange?.(value)}
/>
)}
{showVolume && (
<TuningSliderRow
label="Volume"
valueLabel={`${volume.toFixed(2)}×`}
min={AUDIO_VOLUME_MIN}
max={AUDIO_VOLUME_MAX}
step={0.05}
value={volume}
onChange={(value) => onVolumeChange?.(value)}
/>
)}
{showPitch && (
<TuningSliderRow
label="Pitch"
valueLabel={`${pitch > 0 ? "+" : ""}${pitch} st`}
min={AUDIO_PITCH_MIN}
max={AUDIO_PITCH_MAX}
step={1}
value={pitch}
onChange={(value) => onPitchChange?.(value)}
/>
)}
</div>
</PopoverMenu>
);
}
interface TuningSliderRowProps {
label: string;
valueLabel: string;
min: number;
max: number;
step: number;
value: number;
onChange: (value: number) => void;
}
function TuningSliderRow({
label,
valueLabel,
min,
max,
step,
value,
onChange,
}: TuningSliderRowProps) {
return (
<div>
<div className="mb-1 flex items-center justify-between">
<span className="text-xs font-medium text-base-fg/60">{label}</span>
<span className="text-xs font-medium tabular-nums text-base-fg">
{valueLabel}
</span>
</div>
<SliderV2
min={min}
max={max}
step={step}
value={value}
onChange={onChange}
variant="filled"
/>
</div>
);
}
export function formatSampleRateHz(hz: number): string {
const kilohertz = hz / 1000;
const label = Number.isInteger(kilohertz)
? kilohertz.toString()
: kilohertz.toFixed(1);
return `${label} kHz`;
}
@@ -0,0 +1,126 @@
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faDrum } from "@fortawesome/pro-solid-svg-icons";
import { PopoverMenu } from "@storyteller/ui-popover";
import { SliderV2 } from "@storyteller/ui-sliderv2";
// UI-only bounds; the server accepts any u16 BPM.
export const AUDIO_BPM_MIN = 40;
export const AUDIO_BPM_MAX = 240;
const DEFAULT_BPM = 120;
// Musical keys for audio generation (eg. Suno Sounds).
// NB: There are intentionally no E keys, per product spec.
export const MUSICAL_KEYS: Array<{
value: string;
label: string;
shortLabel: string;
}> = [
{ value: "auto", label: "Auto", shortLabel: "Auto" },
{ value: "c_major", label: "C Major", shortLabel: "C" },
{ value: "c_minor", label: "C Minor", shortLabel: "Cm" },
{ value: "d_major", label: "D Major", shortLabel: "D" },
{ value: "d_minor", label: "D Minor", shortLabel: "Dm" },
{ value: "f_major", label: "F Major", shortLabel: "F" },
{ value: "f_minor", label: "F Minor", shortLabel: "Fm" },
{ value: "g_major", label: "G Major", shortLabel: "G" },
{ value: "g_minor", label: "G Minor", shortLabel: "Gm" },
{ value: "a_major", label: "A Major", shortLabel: "A" },
{ value: "a_minor", label: "A Minor", shortLabel: "Am" },
{ value: "b_major", label: "B Major", shortLabel: "B" },
{ value: "b_minor", label: "B Minor", shortLabel: "Bm" },
];
export interface SoundsSettingsPopoverProps {
showBpm?: boolean;
// Null means "let the model decide" — the field is omitted from the request.
bpm?: number | null;
onBpmChange?: (bpm: number | null) => void;
showMusicalKey?: boolean;
musicalKey?: string;
onMusicalKeyChange?: (key: string) => void;
}
export function SoundsSettingsPopover({
showBpm = false,
bpm = null,
onBpmChange,
showMusicalKey = false,
musicalKey = "auto",
onMusicalKeyChange,
}: SoundsSettingsPopoverProps) {
const keyLabel =
MUSICAL_KEYS.find((entry) => entry.value === musicalKey)?.shortLabel ??
"Auto";
const triggerLabel = [
showBpm ? (bpm === null ? "Auto BPM" : `${bpm} BPM`) : null,
showMusicalKey ? keyLabel : null,
]
.filter(Boolean)
.join(" · ");
return (
<PopoverMenu
mode="default"
panelTitle="Beat & Key"
triggerIcon={<FontAwesomeIcon icon={faDrum} className="h-3.5 w-3.5" />}
triggerLabel={triggerLabel || "Beat"}
>
<div className="w-60 space-y-3.5 pb-0.5">
{showBpm && (
<div>
<div className="mb-1 flex items-center justify-between">
<span className="text-xs font-medium text-base-fg/60">BPM</span>
<div className="flex items-center gap-1.5">
<span className="text-xs font-medium tabular-nums text-base-fg">
{bpm === null ? "Auto" : bpm}
</span>
{bpm !== null && (
<button
type="button"
onClick={() => onBpmChange?.(null)}
className="rounded bg-white/5 px-1.5 py-0.5 text-[10px] font-medium text-base-fg/60 transition-colors hover:bg-white/10 hover:text-base-fg"
>
Auto
</button>
)}
</div>
</div>
<SliderV2
min={AUDIO_BPM_MIN}
max={AUDIO_BPM_MAX}
step={1}
value={bpm ?? DEFAULT_BPM}
onChange={(value) => onBpmChange?.(value)}
variant="filled"
/>
</div>
)}
{showMusicalKey && (
<div>
<div className="mb-1.5 text-xs font-medium text-base-fg/60">
Musical key
</div>
<div className="grid grid-cols-4 gap-1">
{MUSICAL_KEYS.map((entry) => (
<button
key={entry.value}
type="button"
title={entry.label}
onClick={() => onMusicalKeyChange?.(entry.value)}
className={`rounded-md border px-1.5 py-1 text-xs font-medium transition-colors ${
entry.value === musicalKey
? "border-white/30 bg-white/15 text-base-fg"
: "border-transparent bg-white/5 text-base-fg/60 hover:bg-white/10 hover:text-base-fg"
} ${entry.value === "auto" ? "col-span-4" : ""}`}
>
{entry.shortLabel}
</button>
))}
</div>
</div>
)}
</div>
</PopoverMenu>
);
}
@@ -0,0 +1,55 @@
import { ChangeEvent } from "react";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faTags, faXmark } from "@fortawesome/pro-solid-svg-icons";
import { twMerge } from "tailwind-merge";
export interface StylePromptRowProps {
value: string;
onChange: (value: string) => void;
placeholder?: string;
className?: string;
}
// Always-visible secondary prompt row for the style/genre direction
// (Suno's "tags"), rendered directly beneath the main prompt textarea.
export function StylePromptRow({
value,
onChange,
placeholder = "Style: e.g. dreamy synth-pop, female vocals",
className = "",
}: StylePromptRowProps) {
const handleChange = (event: ChangeEvent<HTMLInputElement>) => {
onChange(event.target.value);
};
return (
<div
className={twMerge(
"flex items-center gap-2 border-t border-white/10 px-1 pt-2",
className,
)}
>
<FontAwesomeIcon
icon={faTags}
className="h-3.5 w-3.5 shrink-0 text-base-fg/40"
/>
<input
type="text"
value={value}
onChange={handleChange}
placeholder={placeholder}
className="min-w-0 flex-1 bg-transparent text-sm text-base-fg placeholder:text-base-fg/35 focus:outline-none"
/>
{value.length > 0 && (
<button
type="button"
aria-label="Clear style"
onClick={() => onChange("")}
className="flex h-5 w-5 shrink-0 items-center justify-center rounded text-base-fg/40 transition-colors hover:bg-white/10 hover:text-base-fg"
>
<FontAwesomeIcon icon={faXmark} className="h-3 w-3" />
</button>
)}
</div>
);
}
@@ -186,6 +186,75 @@ export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
setGenerationCount: (generationCount) => set({ generationCount }),
}));
// ----- Audio Prompt Box Store -----
interface PromptAudioStore {
prompt: string;
// Style/genre direction (Suno's "tags").
stylePrompt: string;
// Model id ("suno_music", ...) — audio models come from the omni HTTP API,
// not the ClassyModelSelector store.
selectedModelId: string | null;
// Suno toggles
isInstrumental: boolean;
keepLyrics: boolean;
isLoopable: boolean;
// Suno Sounds beat controls. bpm null = "Auto" (omitted from the request).
bpm: number | null;
musicalKey: string;
// Seed Audio output shaping.
sampleRateHz: number | null;
speed: number;
volume: number;
pitch: number;
referenceAudios: RefAudio[];
referenceImages: RefImage[];
setPrompt: (prompt: string) => void;
setStylePrompt: (stylePrompt: string) => void;
setSelectedModelId: (modelId: string | null) => void;
setIsInstrumental: (value: boolean) => void;
setKeepLyrics: (value: boolean) => void;
setIsLoopable: (value: boolean) => void;
setBpm: (bpm: number | null) => void;
setMusicalKey: (musicalKey: string) => void;
setSampleRateHz: (sampleRateHz: number | null) => void;
setSpeed: (speed: number) => void;
setVolume: (volume: number) => void;
setPitch: (pitch: number) => void;
setReferenceAudios: (audios: RefAudio[]) => void;
setReferenceImages: (images: RefImage[]) => void;
}
export const usePromptAudioStore = create<PromptAudioStore>()((set) => ({
prompt: "",
stylePrompt: "",
selectedModelId: null,
isInstrumental: false,
keepLyrics: false,
isLoopable: false,
bpm: null,
musicalKey: "auto",
sampleRateHz: null,
speed: 1,
volume: 1,
pitch: 0,
referenceAudios: [],
referenceImages: [],
setPrompt: (prompt) => set({ prompt }),
setStylePrompt: (stylePrompt) => set({ stylePrompt }),
setSelectedModelId: (selectedModelId) => set({ selectedModelId }),
setIsInstrumental: (isInstrumental) => set({ isInstrumental }),
setKeepLyrics: (keepLyrics) => set({ keepLyrics }),
setIsLoopable: (isLoopable) => set({ isLoopable }),
setBpm: (bpm) => set({ bpm }),
setMusicalKey: (musicalKey) => set({ musicalKey }),
setSampleRateHz: (sampleRateHz) => set({ sampleRateHz }),
setSpeed: (speed) => set({ speed }),
setVolume: (volume) => set({ volume }),
setPitch: (pitch) => set({ pitch }),
setReferenceAudios: (referenceAudios) => set({ referenceAudios }),
setReferenceImages: (referenceImages) => set({ referenceImages }),
}));
// ----- Edit Prompt Box Store -----
type EditAspectRatio = "auto" | "wide" | "tall" | "square";
@@ -8,6 +8,9 @@
{
"path": "../button-icon-select"
},
{
"path": "../../omni-gen"
},
{
"path": "../../analytics/google-analytics"
},
@@ -49,6 +49,9 @@
{
"path": "../button-icon-select/tsconfig.lib.json"
},
{
"path": "../../omni-gen/tsconfig.lib.json"
},
{
"path": "../../analytics/google-analytics/tsconfig.lib.json"
},
@@ -22,4 +22,5 @@ export enum ModelCreator {
Grok = "Grok",
WorldLabs = "WorldLabs",
Beeble = "Beeble",
Suno = "Suno",
}
@@ -20,6 +20,8 @@ export const MODEL_ID_PREFIX_CREATORS: Array<[string, ModelCreator]> = [
["grok", ModelCreator.Grok],
["happy_horse", ModelCreator.Alibaba],
["qwen", ModelCreator.Alibaba],
["suno", ModelCreator.Suno],
["seed_audio", ModelCreator.Bytedance],
// Beeble SwitchX (background change) has no provider icon — use the ArtCraft
// mark since it's surfaced as an ArtCraft feature.
["switch_x", ModelCreator.ArtCraft],
@@ -26,6 +26,7 @@ const CREATOR_ICON_FILES: Partial<Record<ModelCreator, string>> = {
[ModelCreator.ArtCraft]: "artcraft.svg",
[ModelCreator.Grok]: "grok.svg",
[ModelCreator.WorldLabs]: "worldlabs.svg",
[ModelCreator.Suno]: "suno.svg",
};
export const getCreatorIconPath = (creator: ModelCreator): string => {
@@ -41,6 +41,12 @@ const MODEL_DESCRIPTIONS: Record<string, string> = {
veo_3p1_fast: "Speed-tuned Veo 3.1",
// ── Edit / VFX ──
switch_x: "Swap or relight backgrounds",
// ── Audio models ──
suno_music: "Full songs from a text prompt",
suno_remix: "Remix an existing track",
suno_sounds: "Sound effects with beat control",
suno_sample: "Build a song from a sample",
seed_audio_1p0: "Sound generation with fine tuning",
};
const MODEL_INFOS: Record<string, string> = {
@@ -100,6 +100,15 @@ export const getModelDisplayName = (
seedream_4p5: "Seedream 4.5",
seedream_5_lite: "Seedream 5 Lite",
// Audio — Suno
suno_music: "Suno Music",
suno_remix: "Suno Remix",
suno_sounds: "Suno Sounds",
suno_sample: "Suno Sample",
// Audio — Seed Audio (ByteDance)
seed_audio_1p0: "Seed Audio 1.0",
// Google
veo_2: "Google Veo 2",
veo_3: "Google Veo 3",
+1
View File
@@ -1 +1,2 @@
export * from "./lib/omni-gen-hooks.js";
export * from "./lib/omni-gen-audio.js";
@@ -0,0 +1,140 @@
import { describe, expect, it } from "vitest";
import type { OmniGenAudioModelDetails } from "@storyteller/api";
import { buildAudioRequest, type AudioGenerationSettings } from "./omni-gen-audio";
// Per-model capability fixtures mirroring the backend's audio_models config.
const SUNO_MUSIC: OmniGenAudioModelDetails = {
model: "suno_music",
text_prompt_supported: true,
style_prompt_supported: true,
instrumental_toggle_supported: true,
};
const SUNO_REMIX: OmniGenAudioModelDetails = {
model: "suno_remix",
text_prompt_supported: true,
style_prompt_supported: true,
keep_lyrics_supported: true,
audio_references_supported: true,
audio_references_max: 1,
};
const SUNO_SOUNDS: OmniGenAudioModelDetails = {
model: "suno_sounds",
text_prompt_supported: true,
loopable_toggle_supported: true,
bpm_supported: true,
musical_key_supported: true,
};
const SEED_AUDIO: OmniGenAudioModelDetails = {
model: "seed_audio_1p0",
text_prompt_supported: true,
audio_references_supported: true,
audio_references_max: 3,
image_references_supported: true,
image_references_max: 1,
sample_rate_hz_options: [8000, 16000, 24000, 32000, 44100, 48000],
sample_rate_hz_default: 24000,
speed_supported: true,
volume_supported: true,
pitch_supported: true,
};
const BASE_SETTINGS: AudioGenerationSettings = {
prompt: "a driving synthwave anthem",
stylePrompt: "dreamy synth-pop, female vocals",
audioMediaTokens: ["mf_audio1"],
imageMediaTokens: ["mf_image1"],
isInstrumental: true,
keepLyrics: true,
isLoopable: true,
bpm: 128,
musicalKey: "a_minor",
sampleRateHz: 44100,
speed: 1.25,
volume: 0.8,
pitch: -2,
};
describe("buildAudioRequest capability gating", () => {
it("suno_music sends prompts + instrumental but no refs or beat/tuning params", () => {
const request = buildAudioRequest(SUNO_MUSIC, BASE_SETTINGS);
expect(request.model).toBe("suno_music");
expect(request.prompt).toBe(BASE_SETTINGS.prompt);
expect(request.style_prompt).toBe(BASE_SETTINGS.stylePrompt);
expect(request.is_instrumental).toBe(true);
expect(request.audio_media_tokens).toBeNull();
expect(request.image_media_tokens).toBeNull();
expect(request.keep_lyrics).toBeNull();
expect(request.is_loopable).toBeNull();
expect(request.bpm).toBeNull();
expect(request.musical_key).toBeNull();
expect(request.sample_rate_hz).toBeNull();
expect(request.speed).toBeNull();
expect(request.volume).toBeNull();
expect(request.pitch).toBeNull();
});
it("suno_remix sends the audio reference + keep_lyrics but never style-less extras", () => {
const request = buildAudioRequest(SUNO_REMIX, BASE_SETTINGS);
expect(request.audio_media_tokens).toEqual(["mf_audio1"]);
expect(request.keep_lyrics).toBe(true);
expect(request.image_media_tokens).toBeNull();
expect(request.is_instrumental).toBeNull();
expect(request.bpm).toBeNull();
});
it("suno_sounds sends beat controls but never a style prompt", () => {
const request = buildAudioRequest(SUNO_SOUNDS, BASE_SETTINGS);
expect(request.style_prompt).toBeNull();
expect(request.is_loopable).toBe(true);
expect(request.bpm).toBe(128);
expect(request.musical_key).toBe("a_minor");
expect(request.audio_media_tokens).toBeNull();
});
it("suno_sounds omits bpm when set to Auto (null)", () => {
const request = buildAudioRequest(SUNO_SOUNDS, {
...BASE_SETTINGS,
bpm: null,
});
expect(request.bpm).toBeNull();
expect(request.musical_key).toBe("a_minor");
});
it("seed_audio sends tuning params + refs but never Suno toggles", () => {
const request = buildAudioRequest(SEED_AUDIO, BASE_SETTINGS);
expect(request.sample_rate_hz).toBe(44100);
expect(request.speed).toBe(1.25);
expect(request.volume).toBe(0.8);
expect(request.pitch).toBe(-2);
expect(request.audio_media_tokens).toEqual(["mf_audio1"]);
expect(request.image_media_tokens).toEqual(["mf_image1"]);
expect(request.is_instrumental).toBeNull();
expect(request.keep_lyrics).toBeNull();
expect(request.is_loopable).toBeNull();
expect(request.style_prompt).toBeNull();
});
it("omits empty prompts and reference arrays", () => {
const request = buildAudioRequest(SUNO_MUSIC, {
...BASE_SETTINGS,
prompt: " ",
stylePrompt: "",
audioMediaTokens: [],
imageMediaTokens: [],
});
expect(request.prompt).toBeNull();
expect(request.style_prompt).toBeNull();
expect(request.audio_media_tokens).toBeNull();
});
it("always includes a fresh idempotency token", () => {
const first = buildAudioRequest(SUNO_MUSIC, BASE_SETTINGS);
const second = buildAudioRequest(SUNO_MUSIC, BASE_SETTINGS);
expect(first.idempotency_token).toBeTruthy();
expect(second.idempotency_token).toBeTruthy();
expect(first.idempotency_token).not.toBe(second.idempotency_token);
});
});
@@ -0,0 +1,190 @@
import { useEffect, useRef, useState } from "react";
import { OmniGenApi } from "@storyteller/api";
import type {
OmniGenAudioModelDetails,
OmniGenAudioRequest,
OmniGenMusicalKey,
} from "@storyteller/api";
// Audio request building + enqueue + cost estimation, shared by the webapp
// create-audio page and the desktop PromptBoxAudio (both enqueue over HTTP).
// Server rule (validate_audio_request): these models require exactly one
// audio reference (the remix/sample source). The models API has no
// "required" flag, so mirror it by id.
export const AUDIO_MODELS_REQUIRING_AUDIO_REF = new Set([
"suno_remix",
"suno_sample",
]);
// ── Request params ───────────────────────────────────────────────────────
// Raw UI state. buildAudioRequest gates every field on the selected model's
// capability flags so unsupported settings never reach the API.
export interface AudioGenerationSettings {
prompt: string;
stylePrompt: string;
audioMediaTokens: string[];
imageMediaTokens: string[];
isInstrumental: boolean;
keepLyrics: boolean;
isLoopable: boolean;
bpm: number | null;
musicalKey: string;
sampleRateHz: number | null;
speed: number;
volume: number;
pitch: number;
}
// ── Request builder ──────────────────────────────────────────────────────
// Capability flags are serde-skipped when absent, so only `=== true` counts
// as supported.
const supports = (flag: boolean | null | undefined): boolean => flag === true;
export function buildAudioRequest(
model: OmniGenAudioModelDetails,
settings: AudioGenerationSettings,
): OmniGenAudioRequest {
return {
model: model.model,
idempotency_token: crypto.randomUUID(),
prompt: settings.prompt.trim() || null,
style_prompt:
supports(model.style_prompt_supported) && settings.stylePrompt.trim()
? settings.stylePrompt.trim()
: null,
audio_media_tokens:
supports(model.audio_references_supported) &&
settings.audioMediaTokens.length
? settings.audioMediaTokens
: null,
image_media_tokens:
supports(model.image_references_supported) &&
settings.imageMediaTokens.length
? settings.imageMediaTokens
: null,
keep_lyrics: supports(model.keep_lyrics_supported)
? settings.keepLyrics
: null,
is_instrumental: supports(model.instrumental_toggle_supported)
? settings.isInstrumental
: null,
is_loopable: supports(model.loopable_toggle_supported)
? settings.isLoopable
: null,
bpm:
supports(model.bpm_supported) && settings.bpm !== null
? settings.bpm
: null,
musical_key: supports(model.musical_key_supported)
? (settings.musicalKey as OmniGenMusicalKey)
: null,
sample_rate_hz: model.sample_rate_hz_options?.length
? settings.sampleRateHz
: null,
speed: supports(model.speed_supported) ? settings.speed : null,
volume: supports(model.volume_supported) ? settings.volume : null,
pitch: supports(model.pitch_supported) ? settings.pitch : null,
};
}
// ── Enqueue generation ───────────────────────────────────────────────────
export async function enqueueAudioGeneration(
model: OmniGenAudioModelDetails,
settings: AudioGenerationSettings,
): Promise<{
success: boolean;
// One request can create several jobs (Suno-style multi-clip) — callers
// must treat every token as its own pending generation.
jobTokens: string[];
error?: string;
errorCode?: number;
}> {
const body = buildAudioRequest(model, settings);
try {
const api = new OmniGenApi();
const response = await api.generateAudio(body);
if (response.success && response.inference_job_token) {
const jobTokens = response.all_job_tokens?.length
? response.all_job_tokens
: [response.inference_job_token];
return { success: true, jobTokens };
}
return { success: false, jobTokens: [], error: "Generation failed" };
} catch (err: any) {
return {
success: false,
jobTokens: [],
error: err.message ?? "Request failed",
errorCode: parseAudioHttpStatusCode(err),
};
}
}
// Pull the HTTP status out of an ApiManager error. ApiManager throws
// `Error("HTTP error! status: 402")` on a non-2xx response (the JSON body is
// not surfaced), so callers can special-case codes like 402 Payment Required.
export function parseAudioHttpStatusCode(err: unknown): number | undefined {
const message = err instanceof Error ? err.message : String(err);
const match = /status:\s*(\d+)/.exec(message);
return match ? Number(match[1]) : undefined;
}
// ── Cost estimate hook ───────────────────────────────────────────────────
export interface AudioCostParams {
model: string;
audioReferenceCount: number;
hasImageReference: boolean;
sampleRateHz?: number | null;
}
export function useAudioCostEstimate(params: AudioCostParams): number | null {
const [credits, setCredits] = useState<number | null>(null);
const abortRef = useRef(0);
useEffect(() => {
if (!params.model) {
setCredits(null);
return;
}
const id = ++abortRef.current;
const body: OmniGenAudioRequest = {
model: params.model,
audio_media_tokens: params.audioReferenceCount
? new Array(params.audioReferenceCount).fill("placeholder")
: null,
image_media_tokens: params.hasImageReference ? ["placeholder"] : null,
sample_rate_hz: params.sampleRateHz ?? null,
};
const api = new OmniGenApi();
api.estimateAudioCost(body).then(
(response) => {
if (id !== abortRef.current) return;
if (response.success && response.cost_in_credits != null) {
setCredits(response.cost_in_credits);
} else {
setCredits(null);
}
},
() => {
if (id !== abortRef.current) return;
setCredits(null);
},
);
}, [
params.model,
params.audioReferenceCount,
params.hasImageReference,
params.sampleRateHz,
]);
return credits;
}
@@ -1,6 +1,7 @@
import { useEffect, useState } from "react";
import { OmniGenApi } from "@storyteller/api";
import type {
OmniGenAudioModelDetails,
OmniGenImageModelInfo,
OmniGenVideoModelInfo,
} from "@storyteller/api";
@@ -38,6 +39,11 @@ let videoModelsFetching = false;
let videoModelsError: string | null = null;
let videoModelsListeners: Array<() => void> = [];
let audioModelsCache: OmniGenAudioModelDetails[] | null = null;
let audioModelsFetching = false;
let audioModelsError: string | null = null;
let audioModelsListeners: Array<() => void> = [];
function notifyImageListeners() {
imageModelsListeners.forEach((cb) => cb());
imageModelsListeners = [];
@@ -48,6 +54,11 @@ function notifyVideoListeners() {
videoModelsListeners = [];
}
function notifyAudioListeners() {
audioModelsListeners.forEach((cb) => cb());
audioModelsListeners = [];
}
function fetchImageModelsOnce() {
if (imageModelsCache || imageModelsFetching) return;
imageModelsFetching = true;
@@ -102,6 +113,33 @@ function fetchVideoModelsOnce() {
);
}
function fetchAudioModelsOnce() {
if (audioModelsCache || audioModelsFetching) return;
audioModelsFetching = true;
audioModelsError = null;
const api = new OmniGenApi();
api.getAudioModels().then(
(res) => {
audioModelsFetching = false;
if (res.success && res.models?.length) {
audioModelsCache = res.models.filter((m) => m.is_disabled !== true);
} else {
audioModelsError = "No audio models returned from API";
console.warn("[OmniGen] Audio models response:", res);
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
}
notifyAudioListeners();
},
(err) => {
audioModelsFetching = false;
audioModelsError = err?.message ?? "Failed to fetch audio models";
console.error("[OmniGen] Failed to fetch audio models:", err);
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
notifyAudioListeners();
},
);
}
// ── Hooks ────────────────────────────────────────────────────────────────
export function useOmniGenImageModels(): {
@@ -187,3 +225,45 @@ export function useOmniGenVideoModels(): {
return { models, isLoading, error };
}
export function useOmniGenAudioModels(): {
models: OmniGenAudioModelDetails[];
isLoading: boolean;
error: string | null;
} {
const [models, setModels] = useState<OmniGenAudioModelDetails[]>(
audioModelsCache ?? [],
);
const [isLoading, setIsLoading] = useState(!audioModelsCache);
const [error, setError] = useState<string | null>(audioModelsError);
useEffect(() => {
if (audioModelsCache) {
setModels(audioModelsCache);
setIsLoading(false);
setError(null);
return;
}
const onReady = () => {
setIsLoading(false);
if (audioModelsCache) {
setModels(audioModelsCache);
setError(null);
} else {
setError(audioModelsError);
}
};
audioModelsListeners.push(onReady);
fetchAudioModelsOnce();
return () => {
audioModelsListeners = audioModelsListeners.filter(
(cb) => cb !== onReady,
);
};
}, []);
return { models, isLoading, error };
}
+1
View File
@@ -44,6 +44,7 @@ export function isExternal(id: string): boolean {
id === 'konva' ||
id === 'three' ||
id === 'zustand' ||
id === 'wavesurfer.js' ||
id.startsWith('@fortawesome/')
) {
return true;
+179 -89
View File
File diff suppressed because it is too large Load Diff
+3
View File
@@ -215,6 +215,9 @@
},
{
"path": "./libs/omni-gen"
},
{
"path": "./libs/components/audio-player"
}
]
}