mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-21 09:22:41 -04:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5a58a60475 | ||
|
|
55fab85170 |
@@ -0,0 +1 @@
|
||||
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Suno</title><path d="M16.5 0C20.642 0 24 5.373 24 12h-9c0 6.627-3.358 12-7.5 12C3.358 24 0 18.627 0 12h9c0-6.627 3.358-12 7.5-12z"></path></svg>
|
||||
|
After Width: | Height: | Size: 308 B |
@@ -14,6 +14,7 @@ import Media from "../pages/media";
|
||||
import { ToastContainer } from "../components/toast/toast";
|
||||
import CreateImage from "../pages/create-image";
|
||||
import CreateVideo from "../pages/create-video";
|
||||
import CreateAudio from "../pages/create-audio";
|
||||
import CreateVFX from "../pages/create-vfx";
|
||||
import PageScene from "../pages/pagescene";
|
||||
import PageDraw from "../pages/pagedraw";
|
||||
@@ -153,6 +154,7 @@ export function App() {
|
||||
<Route path="/" element={<Home />} />
|
||||
<Route path="/create-image" element={<CreateImage />} />
|
||||
<Route path="/create-video" element={<CreateVideo />} />
|
||||
<Route path="/create-audio" element={<CreateAudio />} />
|
||||
<Route path="/background-change" element={<CreateVFX />} />
|
||||
<Route path="/moodboard" element={<MoodboardPage />} />
|
||||
<Route path="/edit-3d" element={<PageScene />} />
|
||||
|
||||
+20
-5
@@ -1,7 +1,12 @@
|
||||
import { useCallback, useState } from "react";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import type { IconDefinition } from "@fortawesome/fontawesome-svg-core";
|
||||
import { faCube, faImage, faVideo } from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
faCube,
|
||||
faImage,
|
||||
faMusic,
|
||||
faVideo,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { toast } from "../toast/toast";
|
||||
import { downloadMediaFile } from "../../lib/download-media";
|
||||
import {
|
||||
@@ -23,6 +28,7 @@ import type { GalleryItem } from "./useGalleryData";
|
||||
export interface GalleryItemActions {
|
||||
isVideo: boolean;
|
||||
is3D: boolean;
|
||||
isAudio: boolean;
|
||||
mediaIcon: IconDefinition;
|
||||
mediaLabel: string;
|
||||
modelDisplayName: string | null;
|
||||
@@ -57,15 +63,23 @@ export function useGalleryItemActions(
|
||||
|
||||
const isVideo = item.mediaClass === "video";
|
||||
const is3D = item.mediaClass === "dimensional";
|
||||
const isAudio = item.mediaClass === "audio";
|
||||
// Audio v1: no recreate / make-video — download + share only.
|
||||
const recreateMediaClass: RecreateMediaClass | null = isVideo
|
||||
? "video"
|
||||
: is3D
|
||||
: is3D || isAudio
|
||||
? null
|
||||
: "image";
|
||||
const canMakeVideo = enableMakeVideo && !isVideo && !is3D;
|
||||
const canMakeVideo = enableMakeVideo && !isVideo && !is3D && !isAudio;
|
||||
|
||||
const mediaIcon = isVideo ? faVideo : is3D ? faCube : faImage;
|
||||
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : "Image";
|
||||
const mediaIcon = isVideo
|
||||
? faVideo
|
||||
: is3D
|
||||
? faCube
|
||||
: isAudio
|
||||
? faMusic
|
||||
: faImage;
|
||||
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : isAudio ? "Audio" : "Image";
|
||||
|
||||
const effectiveModelId = modelId ?? item.modelId;
|
||||
const modelDisplayName = effectiveModelId
|
||||
@@ -135,6 +149,7 @@ export function useGalleryItemActions(
|
||||
return {
|
||||
isVideo,
|
||||
is3D,
|
||||
isAudio,
|
||||
mediaIcon,
|
||||
mediaLabel,
|
||||
modelDisplayName,
|
||||
|
||||
+3
-405
@@ -1,406 +1,4 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { JobsApi, JobStatus, MediaFilesApi } from "@storyteller/api";
|
||||
import type { Job, Prompts } from "@storyteller/api";
|
||||
import { getMediaThumbnail, THUMBNAIL_SIZES } from "@storyteller/common";
|
||||
import {
|
||||
getModelDisplayName,
|
||||
getProviderDisplayName,
|
||||
ALL_MODELS_LIST,
|
||||
} from "@storyteller/model-list";
|
||||
import type { GalleryItem } from "./useGalleryData";
|
||||
import { getCachedPrompt, usePrompts } from "../../lib/prompts-cache";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
// The job shapes moved to the shared generation-list lib; re-exported here so
|
||||
// existing import sites keep working.
|
||||
|
||||
// Moved to the shared generation-list lib (used by both webapp and desktop).
|
||||
// Re-exported so every webapp consumer shares the lib's module-level caches.
|
||||
export { useGenerationJobs } from "@storyteller/ui-generation-list";
|
||||
export type { InProgressJob, FailedJob } from "@storyteller/ui-generation-list";
|
||||
import type { InProgressJob, FailedJob } from "@storyteller/ui-generation-list";
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────
|
||||
|
||||
const IN_PROGRESS_STATUSES = new Set([JobStatus.PENDING, JobStatus.STARTED]);
|
||||
const COMPLETED_STATUSES = new Set([JobStatus.COMPLETE_SUCCESS]);
|
||||
const FAILED_STATUSES = new Set([
|
||||
JobStatus.ATTEMPT_FAILED,
|
||||
JobStatus.COMPLETE_FAILURE,
|
||||
JobStatus.DEAD,
|
||||
JobStatus.CANCELLED_BY_USER,
|
||||
JobStatus.CANCELLED_BY_SYSTEM,
|
||||
]);
|
||||
|
||||
const FAILURE_REASON_LABEL: Record<string, string> = {
|
||||
rule_bans_user_image: "Image violates content policy",
|
||||
rule_bans_user_image_with_faces: "Images with faces are not allowed",
|
||||
rule_bans_user_text_prompt: "Text prompt violates content policy",
|
||||
rule_bans_user_content: "Content violates content policy",
|
||||
rule_bans_generated_video: "Generated video flagged by content policy",
|
||||
rule_bans_generated_audio: "Generated audio flagged by content policy",
|
||||
rule_bans_generated_content: "Generated content flagged by content policy",
|
||||
generation_failed: "Generation failed",
|
||||
unknown: "An unknown error occurred",
|
||||
};
|
||||
|
||||
// Cache per-task durations
|
||||
const taskDurationCache = new Map<string, number>();
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function getJobMediaType(job: Job): "image" | "video" | "other" {
|
||||
const cat = job.request.inference_category?.toLowerCase() ?? "";
|
||||
if (cat.includes("video")) return "video";
|
||||
if (cat.includes("image")) return "image";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function getModelLabel(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
|
||||
const modelType =
|
||||
cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
|
||||
const providerKey = cachedPrompt?.maybe_generation_provider ?? modelType;
|
||||
|
||||
const modelDisplay = modelType ? getModelDisplayName(modelType) : undefined;
|
||||
const provider = providerKey
|
||||
? getProviderDisplayName(providerKey.toLowerCase())
|
||||
: undefined;
|
||||
|
||||
if (modelDisplay && provider) return `${modelDisplay} · ${provider}`;
|
||||
return modelDisplay || provider || "Unknown model";
|
||||
}
|
||||
|
||||
function getPrompt(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cached = promptToken
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
return (
|
||||
cached?.maybe_positive_prompt || job.request.maybe_raw_inference_text || ""
|
||||
);
|
||||
}
|
||||
|
||||
function getModelId(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
return cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
|
||||
}
|
||||
|
||||
function jobToInProgress(
|
||||
job: Job,
|
||||
promptsMap: Map<string, Prompts>,
|
||||
): InProgressJob {
|
||||
const now = Date.now();
|
||||
const createdMs = new Date(job.created_at).getTime();
|
||||
const modelType = job.request.maybe_model_type;
|
||||
const isVideo =
|
||||
job.request.inference_category?.toLowerCase().includes("video") ?? false;
|
||||
|
||||
let duration = taskDurationCache.get(job.job_token);
|
||||
if (!duration) {
|
||||
const model = modelType
|
||||
? ALL_MODELS_LIST.find(
|
||||
(m) => m.tauriId === modelType || m.id === modelType,
|
||||
)
|
||||
: undefined;
|
||||
duration = model?.progressBarTime ?? (isVideo ? 900000 : 30000);
|
||||
taskDurationCache.set(job.job_token, duration);
|
||||
}
|
||||
|
||||
const elapsed = now - createdMs;
|
||||
const progress = Math.min(95, (elapsed / duration) * 100);
|
||||
const estimatedTimeLeftMs = Math.max(0, duration - elapsed);
|
||||
|
||||
return {
|
||||
id: job.job_token,
|
||||
prompt: getPrompt(job, promptsMap),
|
||||
modelId: getModelId(job, promptsMap),
|
||||
modelLabel: getModelLabel(job, promptsMap),
|
||||
progress,
|
||||
estimatedTimeLeftMs,
|
||||
createdAt: job.created_at,
|
||||
promptToken: job.request.maybe_prompt_token ?? undefined,
|
||||
mediaClass: isVideo ? "video" : "image",
|
||||
};
|
||||
}
|
||||
|
||||
function jobToFailed(job: Job, promptsMap: Map<string, Prompts>): FailedJob {
|
||||
const failureCategory =
|
||||
job.status.maybe_failure_category_updated ||
|
||||
job.status.maybe_failure_category;
|
||||
const rawMessage =
|
||||
job.status.maybe_failure_message ||
|
||||
job.status.maybe_extra_status_description;
|
||||
const failureReason = failureCategory
|
||||
? FAILURE_REASON_LABEL[failureCategory] || rawMessage || undefined
|
||||
: rawMessage || undefined;
|
||||
const failureMessage =
|
||||
rawMessage && failureCategory !== "unknown" ? rawMessage : undefined;
|
||||
|
||||
const promptToken = job.request.maybe_prompt_token ?? undefined;
|
||||
const promptData = promptToken ? promptsMap.get(promptToken) : undefined;
|
||||
const refImageUrl = pickFirstRefImageUrl(promptData);
|
||||
const inferred = getJobMediaType(job);
|
||||
const mediaClass: "image" | "video" =
|
||||
inferred === "video" ? "video" : "image";
|
||||
|
||||
return {
|
||||
id: job.job_token,
|
||||
prompt: getPrompt(job, promptsMap),
|
||||
modelId: getModelId(job, promptsMap),
|
||||
modelLabel: getModelLabel(job, promptsMap),
|
||||
failureReason,
|
||||
failureMessage,
|
||||
status: job.status.status,
|
||||
createdAt: job.created_at,
|
||||
promptToken,
|
||||
refImageUrl,
|
||||
mediaClass,
|
||||
};
|
||||
}
|
||||
|
||||
// Semantics that aren't still images — videos and audio clips can't be rendered
|
||||
// as the faded backdrop behind the error state, so skip them when picking the
|
||||
// reference to display.
|
||||
const NON_IMAGE_REF_SEMANTICS = new Set(["vid_ref", "audioref"]);
|
||||
|
||||
function pickFirstRefImageUrl(
|
||||
promptData: Prompts | undefined,
|
||||
): string | undefined {
|
||||
const refs = promptData?.maybe_context_images;
|
||||
if (!refs?.length) return undefined;
|
||||
for (const ref of refs) {
|
||||
if (NON_IMAGE_REF_SEMANTICS.has(ref.semantic)) continue;
|
||||
const url = ref.media_links?.cdn_url;
|
||||
if (url) return url;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function jobToGalleryItem(
|
||||
job: Job,
|
||||
promptsMap?: Map<string, Prompts>,
|
||||
): GalleryItem | null {
|
||||
const result = job.maybe_result;
|
||||
if (!result?.entity_token) return null;
|
||||
|
||||
const mediaType = getJobMediaType(job);
|
||||
const mediaClass = mediaType === "video" ? "video" : "image";
|
||||
const thumbnail = getMediaThumbnail(result.media_links, mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
|
||||
return {
|
||||
id: result.entity_token,
|
||||
label: getPrompt(job, promptsMap) || "Generation",
|
||||
thumbnail,
|
||||
fullImage: result.media_links?.cdn_url || null,
|
||||
// Sort by job creation time (not completion time) so the completed card
|
||||
// occupies the same Masonry slot the pending card held — no layout shift.
|
||||
createdAt: job.created_at,
|
||||
mediaClass,
|
||||
modelId: job.request.maybe_model_type ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Expand a single GalleryItem into its batch siblings (if any). */
|
||||
async function expandBatchItems(
|
||||
item: GalleryItem,
|
||||
mediaFilesApi: MediaFilesApi,
|
||||
): Promise<GalleryItem[]> {
|
||||
try {
|
||||
const mediaResponse = await mediaFilesApi.GetMediaFileByToken({
|
||||
mediaFileToken: item.id,
|
||||
});
|
||||
const batchToken = (mediaResponse.data as any)?.maybe_batch_token;
|
||||
if (!batchToken) return [item];
|
||||
|
||||
const batchResponse = await mediaFilesApi.GetMediaFilesByBatchToken({
|
||||
batchToken,
|
||||
});
|
||||
if (!batchResponse.success || !batchResponse.data?.length) return [item];
|
||||
|
||||
return batchResponse.data
|
||||
.map((file: any): GalleryItem | null => {
|
||||
const cdnUrl = file.media_links?.cdn_url;
|
||||
if (!cdnUrl) return null;
|
||||
const thumbnail = getMediaThumbnail(file.media_links, item.mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
return {
|
||||
id: file.token,
|
||||
label: item.label,
|
||||
thumbnail: thumbnail || cdnUrl,
|
||||
fullImage: cdnUrl,
|
||||
createdAt: item.createdAt,
|
||||
mediaClass: item.mediaClass,
|
||||
modelId: item.modelId,
|
||||
batchImageToken: batchToken,
|
||||
};
|
||||
})
|
||||
.filter((i): i is GalleryItem => i !== null);
|
||||
} catch {
|
||||
return [item];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Hook ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export function useGenerationJobs(options: {
|
||||
mediaType: "image" | "video";
|
||||
enabled?: boolean;
|
||||
}) {
|
||||
const { mediaType, enabled = true } = options;
|
||||
const apiRef = useRef(new JobsApi());
|
||||
const mediaApiRef = useRef(new MediaFilesApi());
|
||||
|
||||
const [inProgressJobs, setInProgressJobs] = useState<Job[]>([]);
|
||||
const [failedJobsRaw, setFailedJobsRaw] = useState<Job[]>([]);
|
||||
const [newlyCompleted, setNewlyCompleted] = useState<GalleryItem[]>([]);
|
||||
|
||||
const prevCompletedIdsRef = useRef<Set<string>>(new Set());
|
||||
const initialLoadDoneRef = useRef(false);
|
||||
|
||||
const promptTokens = useMemo(() => {
|
||||
const tokens: string[] = [];
|
||||
for (const j of inProgressJobs) {
|
||||
if (j.request.maybe_prompt_token)
|
||||
tokens.push(j.request.maybe_prompt_token);
|
||||
}
|
||||
for (const j of failedJobsRaw) {
|
||||
if (j.request.maybe_prompt_token)
|
||||
tokens.push(j.request.maybe_prompt_token);
|
||||
}
|
||||
return tokens;
|
||||
}, [inProgressJobs, failedJobsRaw]);
|
||||
const promptsMap = usePrompts(promptTokens);
|
||||
|
||||
const inProgress = useMemo(
|
||||
() =>
|
||||
inProgressJobs
|
||||
.slice()
|
||||
.sort(
|
||||
(a, b) =>
|
||||
new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
|
||||
)
|
||||
.map((j) => jobToInProgress(j, promptsMap)),
|
||||
[inProgressJobs, promptsMap],
|
||||
);
|
||||
|
||||
const failed = useMemo(
|
||||
() =>
|
||||
failedJobsRaw
|
||||
.slice()
|
||||
.sort(
|
||||
(a, b) =>
|
||||
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
|
||||
)
|
||||
.map((j) => jobToFailed(j, promptsMap)),
|
||||
[failedJobsRaw, promptsMap],
|
||||
);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
const response = await apiRef.current.ListRecentJobs();
|
||||
if (!response.success || !response.data) return;
|
||||
|
||||
const jobs: Job[] = response.data;
|
||||
|
||||
// Filter by media type
|
||||
const filtered = jobs.filter((j) => getJobMediaType(j) === mediaType);
|
||||
|
||||
const newInProgress = filtered.filter((j) =>
|
||||
IN_PROGRESS_STATUSES.has(j.status.status),
|
||||
);
|
||||
const newFailed = filtered.filter((j) =>
|
||||
FAILED_STATUSES.has(j.status.status),
|
||||
);
|
||||
|
||||
// Completed
|
||||
const completedJobs = filtered.filter((j) =>
|
||||
COMPLETED_STATUSES.has(j.status.status),
|
||||
);
|
||||
const completedIdSet = new Set(completedJobs.map((j) => j.job_token));
|
||||
|
||||
// Detect newly completed (skip on first load to avoid flooding)
|
||||
let expandedNewItems: GalleryItem[] = [];
|
||||
if (initialLoadDoneRef.current) {
|
||||
const newOnes = completedJobs.filter(
|
||||
(j) => !prevCompletedIdsRef.current.has(j.job_token),
|
||||
);
|
||||
if (newOnes.length > 0) {
|
||||
const items = newOnes
|
||||
.map((j) => jobToGalleryItem(j))
|
||||
.filter((item): item is GalleryItem => item !== null);
|
||||
if (items.length > 0) {
|
||||
// Await expansion so the pending card and its completed replacement
|
||||
// commit in the same React render — no "remove then add" gap.
|
||||
const expanded = await Promise.all(
|
||||
items.map((item) => expandBatchItems(item, mediaApiRef.current)),
|
||||
);
|
||||
expandedNewItems = expanded.flat();
|
||||
}
|
||||
}
|
||||
}
|
||||
initialLoadDoneRef.current = true;
|
||||
prevCompletedIdsRef.current = completedIdSet;
|
||||
|
||||
// Prune duration cache
|
||||
const activeIds = new Set(newInProgress.map((j) => j.job_token));
|
||||
for (const id of taskDurationCache.keys()) {
|
||||
if (!activeIds.has(id)) taskDurationCache.delete(id);
|
||||
}
|
||||
|
||||
if (expandedNewItems.length > 0) {
|
||||
setNewlyCompleted((prev) => {
|
||||
const existingIds = new Set(prev.map((i) => i.id));
|
||||
const fresh = expandedNewItems.filter((i) => !existingIds.has(i.id));
|
||||
return [...fresh, ...prev];
|
||||
});
|
||||
}
|
||||
setInProgressJobs(newInProgress);
|
||||
setFailedJobsRaw(newFailed);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}, [mediaType]);
|
||||
|
||||
// Poll every 5 seconds + listen for task-queue-update events.
|
||||
// Skip entirely when disabled (e.g. user is logged out) — otherwise we'd
|
||||
// hit an authenticated endpoint every 5s for nothing, which on mobile
|
||||
// Safari shows up as periodic main-thread jank during the menu animation.
|
||||
useEffect(() => {
|
||||
if (!enabled) return;
|
||||
load();
|
||||
const intervalId = setInterval(load, 5000);
|
||||
|
||||
const handleTaskUpdate = () => load();
|
||||
window.addEventListener("task-queue-update", handleTaskUpdate);
|
||||
|
||||
return () => {
|
||||
clearInterval(intervalId);
|
||||
window.removeEventListener("task-queue-update", handleTaskUpdate);
|
||||
};
|
||||
}, [load, enabled]);
|
||||
|
||||
const dismissFailed = useCallback(async (jobToken: string) => {
|
||||
try {
|
||||
await apiRef.current.DeleteJobByToken(jobToken);
|
||||
setFailedJobsRaw((prev) => prev.filter((f) => f.job_token !== jobToken));
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
inProgress,
|
||||
failed,
|
||||
newlyCompleted,
|
||||
dismissFailed,
|
||||
refresh: load,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,17 +8,20 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faChevronLeft,
|
||||
faChevronRight,
|
||||
faMusic,
|
||||
faTrashCan,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { addCorsParam, PLACEHOLDER_IMAGES } from "@storyteller/common";
|
||||
import { ActionReminderModal } from "@storyteller/ui-action-reminder-modal";
|
||||
import { Viewer3D } from "@storyteller/ui-viewer-3d";
|
||||
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
|
||||
import useEmblaCarousel from "embla-carousel-react";
|
||||
import type { EmblaOptionsType } from "embla-carousel";
|
||||
import {
|
||||
createPromptData,
|
||||
EMPTY_PROMPT,
|
||||
is3DModelUrl,
|
||||
isAudioUrl,
|
||||
isVideoUrl,
|
||||
type PromptData,
|
||||
} from "./shared";
|
||||
@@ -275,6 +278,9 @@ export function Lightbox({
|
||||
const is3D = selectedImageUrl
|
||||
? is3DModelUrl(selectedImageUrl)
|
||||
: propMediaClass === "dimensional";
|
||||
const isAudio = selectedImageUrl
|
||||
? isAudioUrl(selectedImageUrl) || propMediaClass === "audio"
|
||||
: propMediaClass === "audio";
|
||||
|
||||
// Keyboard navigation
|
||||
useEffect(() => {
|
||||
@@ -312,9 +318,10 @@ export function Lightbox({
|
||||
}
|
||||
}, [selectedMediaToken, mediaFilesApi, onDeleted, onClose]);
|
||||
|
||||
// Audio v1: no recreate / make-video / edit-on-canvas.
|
||||
const recreateMediaClass: "image" | "video" | null = isVideo
|
||||
? "video"
|
||||
: is3D
|
||||
: is3D || isAudio
|
||||
? null
|
||||
: "image";
|
||||
|
||||
@@ -328,14 +335,14 @@ export function Lightbox({
|
||||
);
|
||||
}, [selectedMediaToken, recreateMediaClass, navigate, onClose]);
|
||||
|
||||
const canMakeVideo = !isVideo && !is3D;
|
||||
const canMakeVideo = !isVideo && !is3D && !isAudio;
|
||||
const handleMakeVideo = useCallback(() => {
|
||||
if (!selectedMediaToken || !selectedImageUrl || !canMakeVideo) return;
|
||||
onClose();
|
||||
applyMakeVideoFromImage(selectedMediaToken, selectedImageUrl, navigate);
|
||||
}, [selectedMediaToken, selectedImageUrl, canMakeVideo, navigate, onClose]);
|
||||
|
||||
const canEditOnCanvas = !isVideo && !is3D;
|
||||
const canEditOnCanvas = !isVideo && !is3D && !isAudio;
|
||||
const handleEditOnCanvas = useCallback(() => {
|
||||
if (!selectedMediaToken || !selectedImageUrl || !canEditOnCanvas) return;
|
||||
onClose();
|
||||
@@ -391,6 +398,19 @@ export function Lightbox({
|
||||
>
|
||||
<source src={selectedImageUrl} type="video/mp4" />
|
||||
</video>
|
||||
) : isAudio ? (
|
||||
<div className="flex h-full w-full items-center justify-center px-4 sm:px-8">
|
||||
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-white/[0.04] px-2 py-8">
|
||||
<FontAwesomeIcon
|
||||
icon={faMusic}
|
||||
className="mx-auto mb-5 block text-3xl text-white/20"
|
||||
/>
|
||||
<WaveformAudioPlayer
|
||||
key={selectedImageUrl}
|
||||
src={selectedImageUrl}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex h-full w-full flex-col justify-center">
|
||||
<div
|
||||
@@ -465,7 +485,7 @@ export function Lightbox({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!mediaLoaded && selectedImageUrl && !isVideo && !is3D && (
|
||||
{!mediaLoaded && selectedImageUrl && !isVideo && !is3D && !isAudio && (
|
||||
<div className="absolute inset-0 flex items-center justify-center bg-black">
|
||||
<LoadingSpinner className="h-12 w-12 text-base-fg" />
|
||||
</div>
|
||||
|
||||
@@ -12,6 +12,7 @@ export const COPY_FEEDBACK_DURATION = 1500;
|
||||
|
||||
const VIDEO_EXTENSIONS = [".mp4", ".webm", ".mov", ".avi", ".mkv", ".m4v"];
|
||||
const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz"];
|
||||
const AUDIO_EXTENSIONS = [".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac"];
|
||||
|
||||
export const isVideoUrl = (url: string): boolean =>
|
||||
VIDEO_EXTENSIONS.some((ext) => url.toLowerCase().includes(ext));
|
||||
@@ -19,6 +20,9 @@ export const isVideoUrl = (url: string): boolean =>
|
||||
export const is3DModelUrl = (url: string): boolean =>
|
||||
MODEL_3D_EXTENSIONS.some((ext) => url.toLowerCase().includes(ext));
|
||||
|
||||
export const isAudioUrl = (url: string): boolean =>
|
||||
AUDIO_EXTENSIONS.some((ext) => url.toLowerCase().includes(ext));
|
||||
|
||||
export interface ContextImage {
|
||||
media_links: {
|
||||
cdn_url: string;
|
||||
|
||||
@@ -70,6 +70,10 @@ interface PromptBoxProps {
|
||||
// Model selector (rendered above the toolbar, typically hidden on desktop via lg:hidden)
|
||||
modelSelector?: ReactNode;
|
||||
|
||||
// Secondary prompt row rendered directly under the main textarea
|
||||
// (e.g. the audio style/genre prompt).
|
||||
secondaryPromptRow?: ReactNode;
|
||||
|
||||
// @-mention support (enables colored prompt overlay + autocomplete)
|
||||
mentionItems?: MentionItem[];
|
||||
|
||||
@@ -106,6 +110,7 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
|
||||
onClearAllRefs,
|
||||
mediaReferenceRow,
|
||||
modelSelector,
|
||||
secondaryPromptRow,
|
||||
mentionItems,
|
||||
maxPromptLength,
|
||||
},
|
||||
@@ -355,7 +360,18 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
|
||||
<div className="relative flex flex-col">
|
||||
{isImageRowVisible && renderImagePromptRow()}
|
||||
|
||||
{mediaReferenceRow}
|
||||
{/* When the media row is the topmost band (no image row above it —
|
||||
e.g. the audio page), clip it to rounded top corners; mid-stack
|
||||
(video's start-frame row above) it stays square. */}
|
||||
{mediaReferenceRow && (
|
||||
<div
|
||||
className={twMerge(
|
||||
!isImageRowVisible && "sm:overflow-hidden sm:rounded-t-2xl",
|
||||
)}
|
||||
>
|
||||
{mediaReferenceRow}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div
|
||||
className={twMerge(
|
||||
@@ -543,6 +559,10 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{secondaryPromptRow && (
|
||||
<div className="mt-2">{secondaryPromptRow}</div>
|
||||
)}
|
||||
|
||||
{/* Toolbar */}
|
||||
<div className="mt-2 flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div className="flex flex-wrap items-center gap-1.5 sm:gap-2">
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
faHouse,
|
||||
faVideo,
|
||||
faImage,
|
||||
faMusic,
|
||||
faCube,
|
||||
faFilm,
|
||||
faObjectGroup,
|
||||
@@ -56,6 +57,7 @@ const PRIMARY_ITEMS: NavItem[] = [{ label: "Home", href: "/", icon: faHouse }];
|
||||
const CREATE_ITEMS_STATIC: NavItem[] = [
|
||||
{ label: "Image", href: "/create-image", icon: faImage },
|
||||
{ label: "Video", href: "/create-video", icon: faVideo },
|
||||
{ label: "Audio", href: "/create-audio", icon: faMusic },
|
||||
// Edit Image is hidden from the sidebar for now.
|
||||
{ label: "Edit Image", href: "/edit-image", icon: faPencil },
|
||||
{ label: "Edit 3D", href: "/edit-3d", icon: faCube },
|
||||
|
||||
@@ -111,6 +111,14 @@ export function useStage3DCostEstimate(): void {
|
||||
}, [credits, setEstimatedCreditsForPage]);
|
||||
}
|
||||
|
||||
// ── Audio cost estimate hook ─────────────────────────────────────────────
|
||||
|
||||
// Shared with the desktop PromptBoxAudio via the omni-gen lib.
|
||||
export {
|
||||
useAudioCostEstimate,
|
||||
type AudioCostParams,
|
||||
} from "@storyteller/omni-gen";
|
||||
|
||||
// ── Video cost estimate hook ─────────────────────────────────────────────
|
||||
|
||||
export interface VideoCostParams {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { toast } from "../components/toast/toast";
|
||||
const EXT_BY_MEDIA_CLASS: Record<string, string> = {
|
||||
image: "png",
|
||||
video: "mp4",
|
||||
audio: "mp3",
|
||||
dimensional: "glb",
|
||||
};
|
||||
|
||||
|
||||
@@ -24,6 +24,22 @@ export function resolveModelOption(
|
||||
return options[0];
|
||||
}
|
||||
|
||||
// Number twin of resolveModelOption for numeric option lists (e.g. audio
|
||||
// sample rates). Same semantics: preference when supported, else model
|
||||
// default, else first option; passthrough when the model exposes no options.
|
||||
export function resolveModelNumberOption(
|
||||
preferred: number | null | undefined,
|
||||
options: readonly number[] | null | undefined,
|
||||
modelDefault: number | null | undefined,
|
||||
): number | undefined {
|
||||
if (!options || options.length === 0) {
|
||||
return preferred ?? modelDefault ?? undefined;
|
||||
}
|
||||
if (preferred != null && options.includes(preferred)) return preferred;
|
||||
if (modelDefault != null && options.includes(modelDefault)) return modelDefault;
|
||||
return options[0];
|
||||
}
|
||||
|
||||
// Resolve a preferred batch count against a model's options/limits. Clamps to
|
||||
// [1, max], then snaps to the nearest supported option ≤ the preference (so a
|
||||
// user who wanted 4 keeps the highest the model allows), falling back to the
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { create } from "zustand";
|
||||
import { persist } from "zustand/middleware";
|
||||
import type { RefAudio, RefImage } from "../../components/prompt-box";
|
||||
|
||||
export type AudioUiState = {
|
||||
selectedModelId: string | null;
|
||||
prompt: string;
|
||||
stylePrompt: string;
|
||||
// Suno toggles
|
||||
isInstrumental: boolean;
|
||||
keepLyrics: boolean;
|
||||
isLoopable: boolean;
|
||||
// Suno Sounds beat controls. bpm null = "Auto" (omitted from the request).
|
||||
bpm: number | null;
|
||||
musicalKey: string;
|
||||
// Seed Audio output shaping. sampleRateHz is resolved against the current
|
||||
// model's options at read time (sticky across model switches).
|
||||
sampleRateHz: number | null;
|
||||
speed: number;
|
||||
volume: number;
|
||||
pitch: number;
|
||||
};
|
||||
|
||||
type CreateAudioState = {
|
||||
ui: AudioUiState;
|
||||
referenceAudios: RefAudio[];
|
||||
referenceImages: RefImage[];
|
||||
setUi: (patch: Partial<AudioUiState>) => void;
|
||||
setReferenceAudios: (audios: RefAudio[]) => void;
|
||||
setReferenceImages: (images: RefImage[]) => void;
|
||||
reset: () => void;
|
||||
};
|
||||
|
||||
const DEFAULT_UI: AudioUiState = {
|
||||
selectedModelId: null,
|
||||
prompt: "",
|
||||
stylePrompt: "",
|
||||
isInstrumental: false,
|
||||
keepLyrics: false,
|
||||
isLoopable: false,
|
||||
bpm: null,
|
||||
musicalKey: "auto",
|
||||
sampleRateHz: null,
|
||||
speed: 1,
|
||||
volume: 1,
|
||||
pitch: 0,
|
||||
};
|
||||
|
||||
export const useCreateAudioStore = create<CreateAudioState>()(
|
||||
persist(
|
||||
(set) => ({
|
||||
ui: { ...DEFAULT_UI },
|
||||
referenceAudios: [],
|
||||
referenceImages: [],
|
||||
|
||||
setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })),
|
||||
|
||||
setReferenceAudios: (audios) => set({ referenceAudios: audios }),
|
||||
|
||||
setReferenceImages: (images) => set({ referenceImages: images }),
|
||||
|
||||
reset: () => set({ ui: { ...DEFAULT_UI } }),
|
||||
}),
|
||||
{
|
||||
name: "artcraft-audio-prompt",
|
||||
// Persist prompt + settings so a full page reload keeps the user's
|
||||
// draft. Reference audios/images are excluded: File handles aren't
|
||||
// serializable and uploads can exceed the localStorage quota.
|
||||
partialize: (state) => ({ ui: state.ui }),
|
||||
merge: (persisted, current) => {
|
||||
const persistedUi = (persisted as { ui?: Partial<AudioUiState> } | null)
|
||||
?.ui;
|
||||
return { ...current, ui: { ...current.ui, ...(persistedUi ?? {}) } };
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
@@ -0,0 +1,870 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { FilterMediaClasses } from "@storyteller/api";
|
||||
import type { OmniGenAudioModelDetails } from "@storyteller/api";
|
||||
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
import { Button, ToggleButton } from "@storyteller/ui-button";
|
||||
import { SliderV2 } from "@storyteller/ui-sliderv2";
|
||||
import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal";
|
||||
import {
|
||||
AudioTuningPopover,
|
||||
SoundsSettingsPopover,
|
||||
StylePromptRow,
|
||||
MUSICAL_KEYS,
|
||||
formatSampleRateHz,
|
||||
AUDIO_BPM_MIN,
|
||||
AUDIO_BPM_MAX,
|
||||
AUDIO_SPEED_MIN,
|
||||
AUDIO_SPEED_MAX,
|
||||
AUDIO_VOLUME_MIN,
|
||||
AUDIO_VOLUME_MAX,
|
||||
AUDIO_PITCH_MIN,
|
||||
AUDIO_PITCH_MAX,
|
||||
} from "@storyteller/ui-promptbox";
|
||||
import {
|
||||
PromptBox,
|
||||
ImagePromptRow,
|
||||
MediaReferenceRow,
|
||||
MobilePromptForm,
|
||||
MobileSelectField,
|
||||
MobileFieldButton,
|
||||
SettingsDrawer,
|
||||
DrawerOptionList,
|
||||
DrawerSection,
|
||||
type RefImage,
|
||||
} from "../../components/prompt-box";
|
||||
import {
|
||||
GenerationGallery,
|
||||
useGalleryData,
|
||||
useGenerationJobs,
|
||||
useAuthCheck,
|
||||
usePromptHeight,
|
||||
useLightboxNav,
|
||||
CreateMediaPageShell,
|
||||
} from "../../components/generation-gallery";
|
||||
import { Lightbox } from "../../components/lightbox/lightbox";
|
||||
import { useCreateAudioStore } from "./create-audio-store";
|
||||
import {
|
||||
enqueueAudioGeneration,
|
||||
AUDIO_MODELS_REQUIRING_AUDIO_REF,
|
||||
} from "./generate-audio-api";
|
||||
import type { AudioGenerationSettings } from "./generate-audio-api";
|
||||
import { useAudioCostEstimate } from "../../lib/cost-estimate-api";
|
||||
import { resolveModelNumberOption } from "../../lib/resolve-model-setting";
|
||||
import {
|
||||
useOmniGenAudioModels,
|
||||
OMNI_GENERATE_OUTAGE_MESSAGE,
|
||||
} from "@storyteller/omni-gen";
|
||||
import {
|
||||
getCreatorIconPathForModelId,
|
||||
getModelDescription,
|
||||
getModelInfo,
|
||||
} from "@storyteller/model-list";
|
||||
import { toast } from "../../components/toast/toast";
|
||||
import { useSignupCta } from "../../components/signup-cta-modal";
|
||||
import { useInsufficientCredits } from "../../components/insufficient-credits-modal";
|
||||
import {
|
||||
faMicrophoneLines,
|
||||
faMicrophoneSlash,
|
||||
faRepeat,
|
||||
faSparkles,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_MODEL_ID = "suno_music";
|
||||
|
||||
const AUDIO_FILTER = [FilterMediaClasses.AUDIO];
|
||||
|
||||
const AUDIO_REF_MAX_DURATION_SECONDS = 600;
|
||||
|
||||
const supports = (flag: boolean | null | undefined): boolean => flag === true;
|
||||
|
||||
// Store API model data alongside popover items via a lookup map
|
||||
let _modelLookup = new Map<string, OmniGenAudioModelDetails>();
|
||||
|
||||
function buildModelPopoverItems(
|
||||
models: OmniGenAudioModelDetails[],
|
||||
selectedId: string,
|
||||
): PopoverItem[] {
|
||||
_modelLookup = new Map(models.map((m) => [m.model, m]));
|
||||
return models.map((model) => ({
|
||||
label: model.full_name || model.model,
|
||||
selected: model.model === selectedId,
|
||||
description: getModelDescription(model.model, model.extra_info_short),
|
||||
info: getModelInfo(model.model, model.extra_info) || undefined,
|
||||
icon: (
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(model.model)}
|
||||
alt={`${model.model} logo`}
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
),
|
||||
action: model.model, // use action to carry the model id
|
||||
}));
|
||||
}
|
||||
|
||||
// ── Component ────────────────────────────────────────────────────────────
|
||||
|
||||
export default function CreateAudio() {
|
||||
const { user, authChecked } = useAuthCheck();
|
||||
const { loggedIn, openSignupCta } = useSignupCta();
|
||||
const openInsufficientCredits = useInsufficientCredits();
|
||||
const { promptBoxRef, promptHeight } = usePromptHeight();
|
||||
|
||||
// Fetch models from API
|
||||
const { models: apiModels } = useOmniGenAudioModels();
|
||||
|
||||
// UI state
|
||||
const ui = useCreateAudioStore((s) => s.ui);
|
||||
const setUi = useCreateAudioStore((s) => s.setUi);
|
||||
|
||||
const selectedModel = useMemo(
|
||||
(): OmniGenAudioModelDetails | undefined => {
|
||||
if (!apiModels.length) return undefined;
|
||||
if (ui.selectedModelId) {
|
||||
return (
|
||||
apiModels.find((m) => m.model === ui.selectedModelId) ??
|
||||
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ??
|
||||
apiModels[0]
|
||||
);
|
||||
}
|
||||
return (
|
||||
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0]
|
||||
);
|
||||
},
|
||||
[apiModels, ui.selectedModelId],
|
||||
);
|
||||
|
||||
const prompt = ui.prompt;
|
||||
const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]);
|
||||
const setStylePrompt = useCallback(
|
||||
(v: string) => setUi({ stylePrompt: v }),
|
||||
[setUi],
|
||||
);
|
||||
|
||||
// Capability gating. Flags are serde-skipped when absent — only `true`
|
||||
// counts as supported.
|
||||
const styleSupported = supports(selectedModel?.style_prompt_supported);
|
||||
const hasInstrumental = supports(
|
||||
selectedModel?.instrumental_toggle_supported,
|
||||
);
|
||||
const hasKeepLyrics = supports(selectedModel?.keep_lyrics_supported);
|
||||
const hasLoopable = supports(selectedModel?.loopable_toggle_supported);
|
||||
const hasBpm = supports(selectedModel?.bpm_supported);
|
||||
const hasMusicalKey = supports(selectedModel?.musical_key_supported);
|
||||
const sampleRateOptions = selectedModel?.sample_rate_hz_options ?? null;
|
||||
const hasSpeed = supports(selectedModel?.speed_supported);
|
||||
const hasVolume = supports(selectedModel?.volume_supported);
|
||||
const hasPitch = supports(selectedModel?.pitch_supported);
|
||||
const hasTuning =
|
||||
!!sampleRateOptions?.length || hasSpeed || hasVolume || hasPitch;
|
||||
const audioRefsSupported = supports(
|
||||
selectedModel?.audio_references_supported,
|
||||
);
|
||||
const maxAudioRefs = selectedModel?.audio_references_max ?? 1;
|
||||
const imageRefsSupported = supports(
|
||||
selectedModel?.image_references_supported,
|
||||
);
|
||||
const maxImageRefs = selectedModel?.image_references_max ?? 1;
|
||||
const requiresAudioRef = AUDIO_MODELS_REQUIRING_AUDIO_REF.has(
|
||||
selectedModel?.model ?? "",
|
||||
);
|
||||
|
||||
// Seed Audio output shaping — sticky across model switches; numeric sliders
|
||||
// clamp at read time so persisted values always stay in the valid range.
|
||||
const sampleRateHz =
|
||||
resolveModelNumberOption(
|
||||
ui.sampleRateHz,
|
||||
sampleRateOptions,
|
||||
selectedModel?.sample_rate_hz_default,
|
||||
) ?? null;
|
||||
const speed = clamp(ui.speed, AUDIO_SPEED_MIN, AUDIO_SPEED_MAX);
|
||||
const volume = clamp(ui.volume, AUDIO_VOLUME_MIN, AUDIO_VOLUME_MAX);
|
||||
const pitch = clamp(ui.pitch, AUDIO_PITCH_MIN, AUDIO_PITCH_MAX);
|
||||
|
||||
const [isGenerating, setIsGenerating] = useState(false);
|
||||
const referenceAudios = useCreateAudioStore((s) => s.referenceAudios);
|
||||
const setReferenceAudios = useCreateAudioStore((s) => s.setReferenceAudios);
|
||||
const referenceImages = useCreateAudioStore((s) => s.referenceImages);
|
||||
const setReferenceImages = useCreateAudioStore((s) => s.setReferenceImages);
|
||||
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
|
||||
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
|
||||
const [isBeatDrawerOpen, setIsBeatDrawerOpen] = useState(false);
|
||||
const [isTuningDrawerOpen, setIsTuningDrawerOpen] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (isImagePickerOpen) setPickerSelectedIds([]);
|
||||
}, [isImagePickerOpen]);
|
||||
|
||||
const handlePickerSelect = useCallback(
|
||||
(id: string) => {
|
||||
setPickerSelectedIds((prev) => {
|
||||
if (prev.includes(id)) return prev.filter((x) => x !== id);
|
||||
if (prev.length >= maxImageRefs) {
|
||||
return maxImageRefs === 1 ? [id] : prev;
|
||||
}
|
||||
return [...prev, id];
|
||||
});
|
||||
},
|
||||
[maxImageRefs],
|
||||
);
|
||||
|
||||
// Jobs + gallery
|
||||
const jobs = useGenerationJobs({ mediaType: "audio", enabled: !!user });
|
||||
const gallery = useGalleryData({
|
||||
username: user?.username ?? null,
|
||||
filterMediaClasses: AUDIO_FILTER,
|
||||
excludeUploads: true,
|
||||
});
|
||||
|
||||
const newlyCompletedTokens = useMemo(
|
||||
() => new Set(jobs.newlyCompleted.map((i) => i.id)),
|
||||
[jobs.newlyCompleted],
|
||||
);
|
||||
|
||||
// Lightbox
|
||||
const flatItems = useMemo(() => {
|
||||
const filtered = gallery.items.filter(
|
||||
(i) => !newlyCompletedTokens.has(i.id),
|
||||
);
|
||||
return [...jobs.newlyCompleted, ...filtered];
|
||||
}, [jobs.newlyCompleted, gallery.items, newlyCompletedTokens]);
|
||||
|
||||
const lightbox = useLightboxNav(flatItems);
|
||||
|
||||
const estimatedCredits = useAudioCostEstimate({
|
||||
model: selectedModel?.model ?? "",
|
||||
audioReferenceCount: referenceAudios.length,
|
||||
hasImageReference: referenceImages.length > 0,
|
||||
sampleRateHz: sampleRateOptions?.length ? sampleRateHz : undefined,
|
||||
});
|
||||
|
||||
const modelItems = useMemo(
|
||||
() => buildModelPopoverItems(apiModels, selectedModel?.model ?? ""),
|
||||
[apiModels, selectedModel?.model],
|
||||
);
|
||||
|
||||
const hasContent =
|
||||
jobs.inProgress.length > 0 ||
|
||||
jobs.failed.length > 0 ||
|
||||
jobs.newlyCompleted.length > 0 ||
|
||||
gallery.items.length > 0 ||
|
||||
gallery.isInitialLoading;
|
||||
|
||||
const missingRequiredAudioRef =
|
||||
requiresAudioRef && referenceAudios.length !== 1;
|
||||
const canGenerate =
|
||||
!!prompt.trim() && !isGenerating && !missingRequiredAudioRef;
|
||||
|
||||
// ── Handlers ───────────────────────────────────────────────────────────
|
||||
|
||||
const handleModelChange = useCallback(
|
||||
(item: PopoverItem) => {
|
||||
const model = item.action ? _modelLookup.get(item.action) : undefined;
|
||||
if (!model) return;
|
||||
// Only switch the model — style/toggles/tuning are preserved and
|
||||
// resolved against the new model at read time, so the user's choices
|
||||
// survive model switches instead of resetting to defaults.
|
||||
setUi({ selectedModelId: model.model });
|
||||
},
|
||||
[setUi],
|
||||
);
|
||||
|
||||
// Seed Audio can't combine audio and image references — adding one kind
|
||||
// clears the other so the request is always valid.
|
||||
const handleReferenceAudiosChange = useCallback(
|
||||
(audios: typeof referenceAudios) => {
|
||||
if (audios.length > 0 && referenceImages.length > 0) {
|
||||
setReferenceImages([]);
|
||||
toast.error("Removed image reference — it can't be combined with audio");
|
||||
}
|
||||
setReferenceAudios(audios);
|
||||
},
|
||||
[referenceImages, setReferenceAudios, setReferenceImages],
|
||||
);
|
||||
|
||||
const handleReferenceImagesChange = useCallback(
|
||||
(images: RefImage[]) => {
|
||||
if (images.length > 0 && referenceAudios.length > 0) {
|
||||
setReferenceAudios([]);
|
||||
toast.error("Removed audio reference — it can't be combined with an image");
|
||||
}
|
||||
setReferenceImages(images);
|
||||
},
|
||||
[referenceAudios, setReferenceAudios, setReferenceImages],
|
||||
);
|
||||
|
||||
const handleLibraryImageSelect = useCallback(
|
||||
(items: GalleryItem[]) => {
|
||||
const newImages: RefImage[] = items.slice(0, maxImageRefs).map((item) => ({
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: item.thumbnail || item.fullImage || "",
|
||||
fullUrl: item.fullImage || undefined,
|
||||
file: new File([], "library-image"),
|
||||
mediaToken: item.id,
|
||||
}));
|
||||
handleReferenceImagesChange(newImages);
|
||||
setIsImagePickerOpen(false);
|
||||
},
|
||||
[maxImageRefs, handleReferenceImagesChange],
|
||||
);
|
||||
|
||||
const handleGenerate = useCallback(async () => {
|
||||
if (!loggedIn) {
|
||||
openSignupCta();
|
||||
return;
|
||||
}
|
||||
if (!prompt.trim() || isGenerating || !selectedModel) return;
|
||||
|
||||
if (missingRequiredAudioRef) {
|
||||
toast.error(
|
||||
`${selectedModel.full_name ?? "This model"} needs an audio track to work from — add one first`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
setIsGenerating(true);
|
||||
|
||||
try {
|
||||
const settings: AudioGenerationSettings = {
|
||||
prompt,
|
||||
stylePrompt: ui.stylePrompt,
|
||||
audioMediaTokens: referenceAudios
|
||||
.map((a) => a.mediaToken)
|
||||
.filter((t): t is string => typeof t === "string" && t.length > 0),
|
||||
imageMediaTokens: referenceImages
|
||||
.map((img) => img.mediaToken)
|
||||
.filter((t): t is string => typeof t === "string" && t.length > 0),
|
||||
isInstrumental: ui.isInstrumental,
|
||||
keepLyrics: ui.keepLyrics,
|
||||
isLoopable: ui.isLoopable,
|
||||
bpm: ui.bpm,
|
||||
musicalKey: ui.musicalKey,
|
||||
sampleRateHz,
|
||||
speed,
|
||||
volume,
|
||||
pitch,
|
||||
};
|
||||
|
||||
const result = await enqueueAudioGeneration(selectedModel, settings);
|
||||
|
||||
if (!result.success) {
|
||||
// 402 Payment Required: the user is out of credits. Surface the
|
||||
// upgrade modal instead of an error toast.
|
||||
if (result.errorCode === 402) {
|
||||
openInsufficientCredits();
|
||||
} else if (result.errorCode != null && result.errorCode >= 500) {
|
||||
toast.error(OMNI_GENERATE_OUTAGE_MESSAGE);
|
||||
} else {
|
||||
toast.error(result.error ?? "Failed to start generation");
|
||||
}
|
||||
setIsGenerating(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// The jobs feed (5s poll + this nudge) renders the pending cards — one
|
||||
// per job token (Suno can return several clips per request).
|
||||
window.dispatchEvent(new Event("credits-change"));
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
} catch {
|
||||
toast.error("Network error - please try again");
|
||||
} finally {
|
||||
setIsGenerating(false);
|
||||
}
|
||||
}, [
|
||||
loggedIn,
|
||||
openSignupCta,
|
||||
openInsufficientCredits,
|
||||
prompt,
|
||||
isGenerating,
|
||||
selectedModel,
|
||||
missingRequiredAudioRef,
|
||||
ui.stylePrompt,
|
||||
ui.isInstrumental,
|
||||
ui.keepLyrics,
|
||||
ui.isLoopable,
|
||||
ui.bpm,
|
||||
ui.musicalKey,
|
||||
sampleRateHz,
|
||||
speed,
|
||||
volume,
|
||||
pitch,
|
||||
referenceAudios,
|
||||
referenceImages,
|
||||
]);
|
||||
|
||||
// ── Toolbar pieces (shared between desktop promptbox + fullscreen) ────
|
||||
|
||||
const toggleButtons = (
|
||||
<>
|
||||
{hasInstrumental && (
|
||||
<Tooltip
|
||||
content={ui.isInstrumental ? "Instrumental: ON" : "Instrumental: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={ui.isInstrumental}
|
||||
icon={faMicrophoneSlash}
|
||||
activeIcon={faMicrophoneSlash}
|
||||
label="Instrumental"
|
||||
onClick={() => setUi({ isInstrumental: !ui.isInstrumental })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
{hasKeepLyrics && (
|
||||
<Tooltip
|
||||
content={ui.keepLyrics ? "Keep lyrics: ON" : "Keep lyrics: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={ui.keepLyrics}
|
||||
icon={faMicrophoneLines}
|
||||
activeIcon={faMicrophoneLines}
|
||||
label="Keep lyrics"
|
||||
onClick={() => setUi({ keepLyrics: !ui.keepLyrics })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
{hasLoopable && (
|
||||
<Tooltip
|
||||
content={ui.isLoopable ? "Loop: ON" : "Loop: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={ui.isLoopable}
|
||||
icon={faRepeat}
|
||||
activeIcon={faRepeat}
|
||||
label="Loop"
|
||||
onClick={() => setUi({ isLoopable: !ui.isLoopable })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const leftToolbar = (
|
||||
<>
|
||||
{toggleButtons}
|
||||
{(hasBpm || hasMusicalKey) && (
|
||||
<SoundsSettingsPopover
|
||||
showBpm={hasBpm}
|
||||
bpm={ui.bpm}
|
||||
onBpmChange={(bpm) => setUi({ bpm })}
|
||||
showMusicalKey={hasMusicalKey}
|
||||
musicalKey={ui.musicalKey}
|
||||
onMusicalKeyChange={(musicalKey) => setUi({ musicalKey })}
|
||||
/>
|
||||
)}
|
||||
{hasTuning && (
|
||||
<AudioTuningPopover
|
||||
sampleRateOptions={sampleRateOptions}
|
||||
sampleRateHz={sampleRateHz}
|
||||
onSampleRateChange={(hz) => setUi({ sampleRateHz: hz })}
|
||||
showSpeed={hasSpeed}
|
||||
speed={speed}
|
||||
onSpeedChange={(v) => setUi({ speed: v })}
|
||||
showVolume={hasVolume}
|
||||
volume={volume}
|
||||
onVolumeChange={(v) => setUi({ volume: v })}
|
||||
showPitch={hasPitch}
|
||||
pitch={pitch}
|
||||
onPitchChange={(v) => setUi({ pitch: v })}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const audioReferenceRow = audioRefsSupported ? (
|
||||
<MediaReferenceRow
|
||||
videoSupported={false}
|
||||
audioSupported
|
||||
referenceVideos={[]}
|
||||
onReferenceVideosChange={() => {}}
|
||||
maxVideoCount={0}
|
||||
maxVideoRefDuration={0}
|
||||
referenceAudios={referenceAudios}
|
||||
onReferenceAudiosChange={handleReferenceAudiosChange}
|
||||
maxAudioCount={maxAudioRefs}
|
||||
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
|
||||
/>
|
||||
) : undefined;
|
||||
|
||||
// ── Mobile form ────────────────────────────────────────────────────────
|
||||
|
||||
const beatSummary = [
|
||||
hasBpm ? (ui.bpm === null ? "Auto BPM" : `${ui.bpm} BPM`) : null,
|
||||
hasMusicalKey
|
||||
? (MUSICAL_KEYS.find((k) => k.value === ui.musicalKey)?.label ?? "Auto")
|
||||
: null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
|
||||
const tuningSummary = [
|
||||
sampleRateOptions?.length && sampleRateHz
|
||||
? formatSampleRateHz(sampleRateHz)
|
||||
: null,
|
||||
hasSpeed ? `${speed.toFixed(2)}×` : null,
|
||||
hasPitch && pitch !== 0 ? `${pitch > 0 ? "+" : ""}${pitch} st` : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
|
||||
const mobileForm = (
|
||||
<MobilePromptForm
|
||||
prompt={prompt}
|
||||
onPromptChange={setPrompt}
|
||||
onSubmit={handleGenerate}
|
||||
isSubmitting={isGenerating}
|
||||
credits={estimatedCredits}
|
||||
placeholder="Describe the music or sound you want..."
|
||||
autoAdvance={loggedIn && canGenerate}
|
||||
modelField={
|
||||
<MobileSelectField
|
||||
label="Model"
|
||||
title="Select Model"
|
||||
items={modelItems}
|
||||
onSelect={handleModelChange}
|
||||
/>
|
||||
}
|
||||
frames={
|
||||
imageRefsSupported ? (
|
||||
<ImagePromptRow
|
||||
maxImagePromptCount={maxImageRefs}
|
||||
referenceImages={referenceImages}
|
||||
setReferenceImages={handleReferenceImagesChange}
|
||||
onPickFromLibrary={() => setIsImagePickerOpen(true)}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
mediaRefs={audioReferenceRow}
|
||||
settingsFields={
|
||||
<>
|
||||
{styleSupported && (
|
||||
<div className="rounded-xl border border-ui-panel-border bg-ui-controls px-3 py-1.5">
|
||||
<StylePromptRow
|
||||
value={ui.stylePrompt}
|
||||
onChange={setStylePrompt}
|
||||
className="border-t-0 pt-0"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{(hasInstrumental || hasKeepLyrics || hasLoopable) && (
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{toggleButtons}
|
||||
</div>
|
||||
)}
|
||||
{(hasBpm || hasMusicalKey) && (
|
||||
<>
|
||||
<MobileFieldButton
|
||||
label="Beat & Key"
|
||||
value={beatSummary || "Auto"}
|
||||
onClick={() => setIsBeatDrawerOpen(true)}
|
||||
/>
|
||||
<SettingsDrawer
|
||||
open={isBeatDrawerOpen}
|
||||
onOpenChange={setIsBeatDrawerOpen}
|
||||
title="Beat & Key"
|
||||
>
|
||||
{hasBpm && (
|
||||
<DrawerSection label="BPM">
|
||||
<MobileSliderRow
|
||||
valueLabel={ui.bpm === null ? "Auto" : `${ui.bpm}`}
|
||||
min={AUDIO_BPM_MIN}
|
||||
max={AUDIO_BPM_MAX}
|
||||
step={1}
|
||||
value={ui.bpm ?? 120}
|
||||
onChange={(bpm) => setUi({ bpm })}
|
||||
onReset={
|
||||
ui.bpm !== null ? () => setUi({ bpm: null }) : undefined
|
||||
}
|
||||
/>
|
||||
</DrawerSection>
|
||||
)}
|
||||
{hasMusicalKey && (
|
||||
<DrawerSection label="Musical key">
|
||||
<DrawerOptionList
|
||||
items={MUSICAL_KEYS.map((key) => ({
|
||||
label: key.label,
|
||||
selected: key.value === ui.musicalKey,
|
||||
action: key.value,
|
||||
}))}
|
||||
onSelect={(item) => {
|
||||
if (item.action) setUi({ musicalKey: item.action });
|
||||
}}
|
||||
/>
|
||||
</DrawerSection>
|
||||
)}
|
||||
</SettingsDrawer>
|
||||
</>
|
||||
)}
|
||||
{hasTuning && (
|
||||
<>
|
||||
<MobileFieldButton
|
||||
label="Tuning"
|
||||
value={tuningSummary || "Default"}
|
||||
onClick={() => setIsTuningDrawerOpen(true)}
|
||||
/>
|
||||
<SettingsDrawer
|
||||
open={isTuningDrawerOpen}
|
||||
onOpenChange={setIsTuningDrawerOpen}
|
||||
title="Tuning"
|
||||
>
|
||||
{!!sampleRateOptions?.length && (
|
||||
<DrawerSection label="Sample rate">
|
||||
<DrawerOptionList
|
||||
items={sampleRateOptions.map((hz) => ({
|
||||
label: formatSampleRateHz(hz),
|
||||
selected: hz === sampleRateHz,
|
||||
action: String(hz),
|
||||
}))}
|
||||
onSelect={(item) => {
|
||||
if (item.action)
|
||||
setUi({ sampleRateHz: Number(item.action) });
|
||||
}}
|
||||
/>
|
||||
</DrawerSection>
|
||||
)}
|
||||
{hasSpeed && (
|
||||
<DrawerSection label="Speed">
|
||||
<MobileSliderRow
|
||||
valueLabel={`${speed.toFixed(2)}×`}
|
||||
min={AUDIO_SPEED_MIN}
|
||||
max={AUDIO_SPEED_MAX}
|
||||
step={0.05}
|
||||
value={speed}
|
||||
onChange={(v) => setUi({ speed: v })}
|
||||
/>
|
||||
</DrawerSection>
|
||||
)}
|
||||
{hasVolume && (
|
||||
<DrawerSection label="Volume">
|
||||
<MobileSliderRow
|
||||
valueLabel={`${volume.toFixed(2)}×`}
|
||||
min={AUDIO_VOLUME_MIN}
|
||||
max={AUDIO_VOLUME_MAX}
|
||||
step={0.05}
|
||||
value={volume}
|
||||
onChange={(v) => setUi({ volume: v })}
|
||||
/>
|
||||
</DrawerSection>
|
||||
)}
|
||||
{hasPitch && (
|
||||
<DrawerSection label="Pitch">
|
||||
<MobileSliderRow
|
||||
valueLabel={`${pitch > 0 ? "+" : ""}${pitch} st`}
|
||||
min={AUDIO_PITCH_MIN}
|
||||
max={AUDIO_PITCH_MAX}
|
||||
step={1}
|
||||
value={pitch}
|
||||
onChange={(v) => setUi({ pitch: v })}
|
||||
/>
|
||||
</DrawerSection>
|
||||
)}
|
||||
</SettingsDrawer>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
}
|
||||
/>
|
||||
);
|
||||
|
||||
// ── Render ─────────────────────────────────────────────────────────────
|
||||
|
||||
return (
|
||||
<CreateMediaPageShell
|
||||
title="Create Audio - ArtCraft"
|
||||
description="Generate music and sound effects with ArtCraft"
|
||||
authChecked={authChecked}
|
||||
hasContent={hasContent}
|
||||
emptyStateTitle="Create Audio"
|
||||
emptyStateSubtitle="Describe a song, a sound, or a sample. Hear it in minutes."
|
||||
emptyStateCta={
|
||||
loggedIn ? undefined : (
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={openSignupCta}
|
||||
icon={faSparkles}
|
||||
className="h-12 px-6 text-base font-semibold rounded-full"
|
||||
>
|
||||
Sign up to create
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
bottomOffset={promptHeight + 24}
|
||||
modelItems={modelItems}
|
||||
onModelChange={handleModelChange}
|
||||
promptForm={mobileForm}
|
||||
gridContent={
|
||||
<GenerationGallery
|
||||
inProgressJobs={jobs.inProgress}
|
||||
failedJobs={jobs.failed}
|
||||
onDismissFailed={jobs.dismissFailed}
|
||||
newlyCompletedItems={jobs.newlyCompleted}
|
||||
galleryItems={gallery.items}
|
||||
newlyCompletedTokens={newlyCompletedTokens}
|
||||
hasMore={gallery.hasMore}
|
||||
isLoading={gallery.isLoading}
|
||||
isInitialLoading={gallery.isInitialLoading}
|
||||
onLoadMore={gallery.loadMore}
|
||||
onGalleryItemClick={lightbox.handleGalleryItemClick}
|
||||
/>
|
||||
}
|
||||
promptBox={
|
||||
<div
|
||||
ref={promptBoxRef}
|
||||
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
|
||||
style={{
|
||||
animationDelay: "150ms",
|
||||
left: "var(--ac-sidebar-offset, 0px)",
|
||||
}}
|
||||
>
|
||||
<PromptBox
|
||||
prompt={prompt}
|
||||
onPromptChange={setPrompt}
|
||||
onSubmit={handleGenerate}
|
||||
isSubmitting={isGenerating}
|
||||
disabled={!canGenerate}
|
||||
credits={estimatedCredits}
|
||||
placeholder="Describe the music or sound you want..."
|
||||
supportsImagePrompts={imageRefsSupported}
|
||||
maxImagePromptCount={maxImageRefs}
|
||||
referenceImages={referenceImages}
|
||||
onReferenceImagesChange={handleReferenceImagesChange}
|
||||
onPickFromLibrary={() => setIsImagePickerOpen(true)}
|
||||
mediaReferenceRow={audioReferenceRow}
|
||||
secondaryPromptRow={
|
||||
styleSupported ? (
|
||||
<StylePromptRow
|
||||
value={ui.stylePrompt}
|
||||
onChange={setStylePrompt}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
modelSelector={
|
||||
<Tooltip
|
||||
content="Model"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick
|
||||
>
|
||||
<PopoverMenu
|
||||
items={modelItems}
|
||||
onSelect={handleModelChange}
|
||||
mode="toggle"
|
||||
panelTitle="Select Model"
|
||||
panelClassName="w-[360px]"
|
||||
richList
|
||||
triggerIcon={
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(
|
||||
selectedModel?.model ?? "",
|
||||
)}
|
||||
alt=""
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
}
|
||||
leftToolbar={leftToolbar}
|
||||
rightToolbar={
|
||||
missingRequiredAudioRef ? (
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-red-500 animate-pulse">
|
||||
Audio track required
|
||||
</span>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
modals={
|
||||
<>
|
||||
<GalleryModal
|
||||
mode="select"
|
||||
isOpen={isImagePickerOpen}
|
||||
onClose={() => setIsImagePickerOpen(false)}
|
||||
selectedItemIds={pickerSelectedIds}
|
||||
onSelectItem={handlePickerSelect}
|
||||
maxSelections={maxImageRefs}
|
||||
onUseSelected={handleLibraryImageSelect}
|
||||
forceFilter="image"
|
||||
hideFilter
|
||||
/>
|
||||
<Lightbox
|
||||
isOpen={lightbox.lightboxOpen}
|
||||
onClose={lightbox.closeLightbox}
|
||||
mediaToken={lightbox.lightboxItem?.id}
|
||||
cdnUrl={lightbox.lightboxItem?.fullImage}
|
||||
mediaClass={lightbox.lightboxItem?.mediaClass}
|
||||
batchImageToken={lightbox.lightboxItem?.batchImageToken}
|
||||
showBatchCarousel={false}
|
||||
onNavigatePrev={lightbox.navigatePrev}
|
||||
onNavigateNext={lightbox.navigateNext}
|
||||
onDeleted={gallery.removeItem}
|
||||
/>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.min(Math.max(value, min), max);
|
||||
}
|
||||
|
||||
interface MobileSliderRowProps {
|
||||
valueLabel: string;
|
||||
min: number;
|
||||
max: number;
|
||||
step: number;
|
||||
value: number;
|
||||
onChange: (value: number) => void;
|
||||
onReset?: () => void;
|
||||
}
|
||||
|
||||
function MobileSliderRow({
|
||||
valueLabel,
|
||||
min,
|
||||
max,
|
||||
step,
|
||||
value,
|
||||
onChange,
|
||||
onReset,
|
||||
}: MobileSliderRowProps) {
|
||||
return (
|
||||
<div className="px-1 py-1.5">
|
||||
<div className="mb-1.5 flex items-center justify-between">
|
||||
<span className="text-sm font-medium tabular-nums text-base-fg">
|
||||
{valueLabel}
|
||||
</span>
|
||||
{onReset && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onReset}
|
||||
className="rounded bg-white/5 px-2 py-0.5 text-xs font-medium text-base-fg/60 transition-colors hover:bg-white/10 hover:text-base-fg"
|
||||
>
|
||||
Auto
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<SliderV2
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
variant="filled"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// The audio request builder + enqueue live in the shared omni-gen lib (used
|
||||
// by both this page and the desktop PromptBoxAudio). Re-exported so this
|
||||
// page keeps the same local-module structure as create-image / create-video.
|
||||
export {
|
||||
buildAudioRequest,
|
||||
enqueueAudioGeneration,
|
||||
AUDIO_MODELS_REQUIRING_AUDIO_REF,
|
||||
} from "@storyteller/omni-gen";
|
||||
export type { AudioGenerationSettings } from "@storyteller/omni-gen";
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from "./create-audio";
|
||||
@@ -4,6 +4,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faImage,
|
||||
faVideo,
|
||||
faMusic,
|
||||
faWandMagicSparkles,
|
||||
faCube,
|
||||
faFilm,
|
||||
@@ -47,6 +48,15 @@ const APPS: AppCard[] = [
|
||||
iconBg: "bg-purple-500/25 border-purple-400/30",
|
||||
iconColor: "text-purple-300",
|
||||
},
|
||||
{
|
||||
label: "Audio",
|
||||
description: "Generate songs and sound effects from prompts.",
|
||||
href: "/create-audio",
|
||||
icon: faMusic,
|
||||
accent: "from-pink-500/20 to-pink-500/0",
|
||||
iconBg: "bg-pink-500/25 border-pink-400/30",
|
||||
iconColor: "text-pink-300",
|
||||
},
|
||||
{
|
||||
label: "Edit 3D",
|
||||
description: "Compose 3D scenes and render with AI cameras.",
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
faBorderAll,
|
||||
faCube,
|
||||
faImage,
|
||||
faMusic,
|
||||
faVideo,
|
||||
faPencil,
|
||||
faTrashCan,
|
||||
@@ -59,12 +60,14 @@ const FILTERS = [
|
||||
{ id: "all", label: "All", icon: faBorderAll, route: "/library" },
|
||||
{ id: "image", label: "Images", icon: faImage, route: "/library/images" },
|
||||
{ id: "video", label: "Videos", icon: faVideo, route: "/library/videos" },
|
||||
{ id: "audio", label: "Audio", icon: faMusic, route: "/library/audio" },
|
||||
{ id: "meshes", label: "Meshes", icon: faCube, route: "/library/meshes" },
|
||||
];
|
||||
|
||||
const ROUTE_TO_FILTER: Record<string, string> = {
|
||||
images: "image",
|
||||
videos: "video",
|
||||
audio: "audio",
|
||||
meshes: "meshes",
|
||||
};
|
||||
|
||||
@@ -76,12 +79,15 @@ const getFilterMediaClass = (
|
||||
return [FilterMediaClasses.IMAGE];
|
||||
case "video":
|
||||
return [FilterMediaClasses.VIDEO];
|
||||
case "audio":
|
||||
return [FilterMediaClasses.AUDIO];
|
||||
case "meshes":
|
||||
return [FilterMediaClasses.DIMENSIONAL];
|
||||
default:
|
||||
return [
|
||||
FilterMediaClasses.IMAGE,
|
||||
FilterMediaClasses.VIDEO,
|
||||
FilterMediaClasses.AUDIO,
|
||||
FilterMediaClasses.DIMENSIONAL,
|
||||
];
|
||||
}
|
||||
@@ -1547,7 +1553,9 @@ function BulkThumb({ item }: { item: GalleryItem }) {
|
||||
? faVideo
|
||||
: item.mediaClass === "dimensional"
|
||||
? faCube
|
||||
: faImage;
|
||||
: item.mediaClass === "audio"
|
||||
? faMusic
|
||||
: faImage;
|
||||
const showImage = !!item.thumbnail && !failed;
|
||||
return (
|
||||
<div className="-ml-2 h-8 w-8 flex-shrink-0 overflow-hidden rounded border-2 border-ui-panel bg-black/30 first:ml-0">
|
||||
|
||||
@@ -79,6 +79,9 @@
|
||||
{
|
||||
"path": "../../libs/components/switch/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/audio-player/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/viewer-3d/tsconfig.lib.json"
|
||||
},
|
||||
|
||||
@@ -65,6 +65,9 @@
|
||||
{
|
||||
"path": "../../libs/components/generation-list"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/audio-player"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/tab-selector"
|
||||
},
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Suno</title><path d="M16.5 0C20.642 0 24 5.373 24 12h-9c0 6.627-3.358 12-7.5 12C3.358 24 0 18.627 0 12h9c0-6.627 3.358-12 7.5-12z"></path></svg>
|
||||
|
After Width: | Height: | Size: 308 B |
@@ -0,0 +1 @@
|
||||
<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Suno</title><path d="M4 9.25c.69 0 1.25.56 1.25 1.25v3a1.25 1.25 0 1 1-2.5 0v-3c0-.69.56-1.25 1.25-1.25zM8 5.5c.69 0 1.25.56 1.25 1.25v10.5a1.25 1.25 0 1 1-2.5 0V6.75c0-.69.56-1.25 1.25-1.25zM12 2.75c.69 0 1.25.56 1.25 1.25v16a1.25 1.25 0 1 1-2.5 0V4c0-.69.56-1.25 1.25-1.25zM16 6.5c.69 0 1.25.56 1.25 1.25v8.5a1.25 1.25 0 1 1-2.5 0v-8.5c0-.69.56-1.25 1.25-1.25zM20 9.75c.69 0 1.25.56 1.25 1.25v2a1.25 1.25 0 1 1-2.5 0v-2c0-.69.56-1.25 1.25-1.25z"></path></svg>
|
||||
|
After Width: | Height: | Size: 626 B |
+4
-2
@@ -116,13 +116,15 @@ function ItemActions({
|
||||
|
||||
const isVideo = item.mediaClass === "video";
|
||||
const is3D = item.mediaClass === "dimensional";
|
||||
const isAudio = item.mediaClass === "audio";
|
||||
// Audio v1: no recreate / make-video — share + download only.
|
||||
const recreateMediaClass: "image" | "video" | null = isVideo
|
||||
? "video"
|
||||
: is3D
|
||||
: is3D || isAudio
|
||||
? null
|
||||
: "image";
|
||||
const canRecreate = !!promptToken && !!recreateMediaClass;
|
||||
const canMakeVideo = !!enableMakeVideo && !isVideo && !is3D;
|
||||
const canMakeVideo = !!enableMakeVideo && !isVideo && !is3D && !isAudio;
|
||||
|
||||
const buttonClass =
|
||||
variant === "card"
|
||||
|
||||
@@ -357,6 +357,8 @@ export const TopBar = ({ pageName }: Props) => {
|
||||
return "Create Image";
|
||||
case "VIDEO":
|
||||
return "Create Video";
|
||||
case "AUDIO":
|
||||
return "Create Audio";
|
||||
case "EDIT":
|
||||
return "Edit Image";
|
||||
case "VIDEO_FRAME_EXTRACTOR":
|
||||
@@ -499,7 +501,8 @@ export const TopBar = ({ pageName }: Props) => {
|
||||
<div className="flex justify-end gap-2" data-tauri-drag-region>
|
||||
<div className="no-drag flex items-center gap-1.5">
|
||||
{(tabStore.activeTabId === "IMAGE" ||
|
||||
tabStore.activeTabId === "VIDEO") && <GalleryViewToggle />}
|
||||
tabStore.activeTabId === "VIDEO" ||
|
||||
tabStore.activeTabId === "AUDIO") && <GalleryViewToggle />}
|
||||
<PopoverMenu
|
||||
position="bottom"
|
||||
align="center"
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
faCube,
|
||||
faFilm,
|
||||
faImage,
|
||||
faMusic,
|
||||
faDroplet,
|
||||
faPhotoFilm,
|
||||
faGlobe,
|
||||
@@ -23,6 +24,7 @@ import { useTabStore, TabId } from "~/pages/Stores/TabState";
|
||||
export type AppId =
|
||||
| "IMAGE"
|
||||
| "VIDEO"
|
||||
| "AUDIO"
|
||||
| "EDIT"
|
||||
| "2D"
|
||||
| "3D"
|
||||
@@ -58,6 +60,11 @@ export const APP_DESCRIPTORS: AppDescriptor[] = [
|
||||
label: "Create Video",
|
||||
icon: faFilm,
|
||||
},
|
||||
{
|
||||
id: "AUDIO",
|
||||
label: "Create Audio",
|
||||
icon: faMusic,
|
||||
},
|
||||
{
|
||||
id: "2D",
|
||||
label: "Image Editor",
|
||||
@@ -113,6 +120,11 @@ const APP_CARD_PALETTES: Record<string, AppCardPalette> = {
|
||||
iconBg: "bg-amber-500/20 border-amber-400/30",
|
||||
iconColor: "text-amber-300",
|
||||
},
|
||||
"create-audio": {
|
||||
accent: "from-pink-500/20 to-pink-500/0",
|
||||
iconBg: "bg-pink-500/20 border-pink-400/30",
|
||||
iconColor: "text-pink-300",
|
||||
},
|
||||
"image-to-3d-object": {
|
||||
accent: "from-emerald-500/20 to-emerald-500/0",
|
||||
iconBg: "bg-emerald-500/20 border-emerald-400/30",
|
||||
@@ -208,6 +220,15 @@ export const ALL_APPS: FullAppItem[] = [
|
||||
action: "VIDEO",
|
||||
color: "bg-amber-500/40",
|
||||
},
|
||||
{
|
||||
id: "create-audio",
|
||||
label: "Create Audio",
|
||||
description: "Generate music and sound effects",
|
||||
icon: faMusic,
|
||||
category: "generate",
|
||||
action: "AUDIO",
|
||||
color: "bg-pink-500/40",
|
||||
},
|
||||
{
|
||||
id: "image-to-3d-object",
|
||||
label: "Image to 3D Object",
|
||||
@@ -398,6 +419,7 @@ export const goToApp = (action?: string) => {
|
||||
[
|
||||
"IMAGE",
|
||||
"VIDEO",
|
||||
"AUDIO",
|
||||
"2D",
|
||||
"3D",
|
||||
"VIDEO_FRAME_EXTRACTOR",
|
||||
|
||||
@@ -60,6 +60,7 @@ import { AppsIndexPage } from "./PageApps/AppsIndexPage";
|
||||
import PageDraw from "./PageDraw/PageDraw";
|
||||
import TextToImage from "./PageImage/TextToImage";
|
||||
import ImageToVideo from "./PageVideo/ImageToVideo";
|
||||
import CreateAudio from "./PageAudio/CreateAudio";
|
||||
import { VideoFrameExtractor } from "./PageVideoFrameExtractor";
|
||||
import { VideoWatermarkRemover } from "./PageVideoWatermarkRemover";
|
||||
import { ImageWatermarkRemover } from "./PageImageWatermarkRemover";
|
||||
@@ -247,6 +248,12 @@ const TabBody = ({ sceneToken }: { sceneToken?: string }) => {
|
||||
<ImageToVideo />
|
||||
</div>
|
||||
);
|
||||
case "AUDIO":
|
||||
return (
|
||||
<div>
|
||||
<CreateAudio />
|
||||
</div>
|
||||
);
|
||||
case "VIDEO_FRAME_EXTRACTOR":
|
||||
return (
|
||||
<div>
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { PromptBoxAudio, usePromptAudioStore } from "@storyteller/ui-promptbox";
|
||||
import {
|
||||
UploadAudioMedia,
|
||||
UploadImageMedia,
|
||||
FilterMediaClasses,
|
||||
} from "@storyteller/api";
|
||||
import { useOmniGenAudioModels, useAudioCostEstimate } from "@storyteller/omni-gen";
|
||||
import BackgroundGallery from "../PageImage/BackgroundGallery";
|
||||
import {
|
||||
galleryModalLightboxImage,
|
||||
galleryModalLightboxMediaId,
|
||||
galleryModalLightboxVisible,
|
||||
galleryModalLightboxNavPrev,
|
||||
galleryModalLightboxNavNext,
|
||||
} from "@storyteller/ui-gallery-modal";
|
||||
import { HelpMenuButton } from "@storyteller/ui-help-menu";
|
||||
import {
|
||||
useGalleryData,
|
||||
useGenerationJobs,
|
||||
type GalleryItem,
|
||||
} from "@storyteller/ui-generation-list";
|
||||
import { useDesktopUsername } from "~/components/generation-feed/useDesktopUsername";
|
||||
import { DesktopCreatePageShell } from "~/components/generation-feed/DesktopCreatePageShell";
|
||||
import { DesktopGenerationGallery } from "~/components/generation-feed/DesktopGenerationGallery";
|
||||
|
||||
const AUDIO_FILTER = [FilterMediaClasses.AUDIO];
|
||||
|
||||
const CreateAudio = () => {
|
||||
const { models } = useOmniGenAudioModels();
|
||||
const promptContentRef = useRef<HTMLDivElement>(null);
|
||||
const [promptHeight, setPromptHeight] = useState<number>(138);
|
||||
|
||||
// Cost estimate inputs come from the shared audio prompt store (the
|
||||
// promptbox owns the settings; the page only observes them).
|
||||
const selectedModelId = usePromptAudioStore((s) => s.selectedModelId);
|
||||
const referenceAudios = usePromptAudioStore((s) => s.referenceAudios);
|
||||
const referenceImages = usePromptAudioStore((s) => s.referenceImages);
|
||||
const selectedModel = useMemo(
|
||||
() =>
|
||||
models.find((m) => m.model === selectedModelId) ??
|
||||
models.find((m) => m.model === "suno_music") ??
|
||||
models[0],
|
||||
[models, selectedModelId],
|
||||
);
|
||||
const audioCredits = useAudioCostEstimate({
|
||||
model: selectedModel?.model ?? "",
|
||||
audioReferenceCount: referenceAudios.length,
|
||||
hasImageReference: referenceImages.length > 0,
|
||||
sampleRateHz: selectedModel?.sample_rate_hz_options?.length
|
||||
? (selectedModel.sample_rate_hz_default ?? undefined)
|
||||
: undefined,
|
||||
});
|
||||
|
||||
// Track the promptbox height so the feed can pad past it.
|
||||
useEffect(() => {
|
||||
const el = promptContentRef.current;
|
||||
if (!el || typeof ResizeObserver === "undefined") return;
|
||||
const update = () => setPromptHeight(el.offsetHeight);
|
||||
update();
|
||||
const ro = new ResizeObserver(() => update());
|
||||
ro.observe(el);
|
||||
return () => ro.disconnect();
|
||||
}, []);
|
||||
|
||||
// The merged generation feed. Audio enqueues over HTTP (no Tauri task
|
||||
// queue), so in-progress/failed come from the shared jobs poller and the
|
||||
// completed history from the library — exactly like the webapp.
|
||||
const username = useDesktopUsername();
|
||||
const feed = useGenerationJobs({ mediaType: "audio", enabled: !!username });
|
||||
const gallery = useGalleryData({
|
||||
username,
|
||||
filterMediaClasses: AUDIO_FILTER,
|
||||
excludeUploads: true,
|
||||
});
|
||||
|
||||
// Content only — while the gallery is still loading the shell keeps the
|
||||
// hero + background up as a splash and fades them out when items land.
|
||||
const hasContent =
|
||||
feed.inProgress.length > 0 ||
|
||||
feed.failed.length > 0 ||
|
||||
feed.newlyCompleted.length > 0 ||
|
||||
gallery.items.length > 0;
|
||||
|
||||
const newlyCompletedTokens = useMemo(
|
||||
() => new Set(feed.newlyCompleted.map((item) => item.id)),
|
||||
[feed.newlyCompleted],
|
||||
);
|
||||
|
||||
// Flat, time-sorted completed list driving lightbox prev/next navigation.
|
||||
const flatCompleted = useMemo(() => {
|
||||
const seen = new Set<string>();
|
||||
const merged: GalleryItem[] = [];
|
||||
for (const item of feed.newlyCompleted) {
|
||||
if (!seen.has(item.id)) {
|
||||
seen.add(item.id);
|
||||
merged.push(item);
|
||||
}
|
||||
}
|
||||
for (const item of gallery.items) {
|
||||
if (!seen.has(item.id)) {
|
||||
seen.add(item.id);
|
||||
merged.push(item);
|
||||
}
|
||||
}
|
||||
merged.sort(
|
||||
(a, b) =>
|
||||
new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
|
||||
);
|
||||
return merged;
|
||||
}, [feed.newlyCompleted, gallery.items]);
|
||||
|
||||
const flatCompletedRef = useRef(flatCompleted);
|
||||
flatCompletedRef.current = flatCompleted;
|
||||
|
||||
// Open a completed row in the global lightbox (rendered by TopBar's
|
||||
// gallery modal); prev/next walk the merged feed order.
|
||||
const openInLightbox = useCallback((item: GalleryItem) => {
|
||||
const list = flatCompletedRef.current;
|
||||
const index = list.findIndex((i) => i.id === item.id);
|
||||
galleryModalLightboxNavPrev.value =
|
||||
index > 0 ? () => openInLightbox(list[index - 1]) : null;
|
||||
galleryModalLightboxNavNext.value =
|
||||
index >= 0 && index < list.length - 1
|
||||
? () => openInLightbox(list[index + 1])
|
||||
: null;
|
||||
galleryModalLightboxMediaId.value = item.id;
|
||||
galleryModalLightboxImage.value = {
|
||||
id: item.id,
|
||||
label: item.label,
|
||||
thumbnail: item.thumbnail,
|
||||
fullImage: item.fullImage,
|
||||
createdAt: item.createdAt,
|
||||
mediaClass: item.mediaClass,
|
||||
};
|
||||
galleryModalLightboxVisible.value = true;
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<DesktopCreatePageShell
|
||||
hasContent={hasContent}
|
||||
emptyStateTitle="Create Audio"
|
||||
emptyStateSubtitle="Describe a song, a sound, or a sample"
|
||||
background={<BackgroundGallery />}
|
||||
bottomOffset={promptHeight + 40}
|
||||
listContent={
|
||||
<DesktopGenerationGallery
|
||||
inProgressJobs={feed.inProgress}
|
||||
failedJobs={feed.failed}
|
||||
onDismissFailed={feed.dismissFailed}
|
||||
newlyCompletedItems={feed.newlyCompleted}
|
||||
galleryItems={gallery.items}
|
||||
newlyCompletedTokens={newlyCompletedTokens}
|
||||
hasMore={gallery.hasMore}
|
||||
isLoading={gallery.isLoading}
|
||||
isInitialLoading={gallery.isInitialLoading}
|
||||
onLoadMore={gallery.loadMore}
|
||||
onGalleryItemClick={openInLightbox}
|
||||
/>
|
||||
}
|
||||
promptBox={
|
||||
<div className="fixed bottom-4 left-1/2 z-20 w-full max-w-5xl -translate-x-1/2 px-2 sm:px-4">
|
||||
<div ref={promptContentRef}>
|
||||
<PromptBoxAudio
|
||||
models={models}
|
||||
uploadAudio={UploadAudioMedia}
|
||||
uploadImage={UploadImageMedia}
|
||||
credits={audioCredits}
|
||||
onEnqueuePressed={async () => {
|
||||
// Nudge the shared jobs poller so the pending cards appear
|
||||
// immediately (one per created job token).
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
window.dispatchEvent(new Event("credits-change"));
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
bottomRight={<HelpMenuButton />}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default CreateAudio;
|
||||
@@ -7,6 +7,7 @@ export type TabId =
|
||||
| "VIDEO"
|
||||
| "EDIT"
|
||||
| "IMAGE"
|
||||
| "AUDIO"
|
||||
| "APPS"
|
||||
| "VIDEO_FRAME_EXTRACTOR"
|
||||
| "VIDEO_WATERMARK_REMOVAL"
|
||||
|
||||
@@ -75,7 +75,6 @@
|
||||
"tailwind-merge": "^2.2.2",
|
||||
"terser": "^5.30.3",
|
||||
"uuid": "^9.0.1",
|
||||
"wavesurfer.js": "^7.7.11",
|
||||
"webm-writer": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -37,6 +37,9 @@
|
||||
{
|
||||
"path": "../../libs/components/vfx"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/omni-gen"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/slider-v2"
|
||||
},
|
||||
|
||||
@@ -73,6 +73,89 @@ export interface OmniGenVideoGenerateResponse {
|
||||
inference_job_token: string;
|
||||
}
|
||||
|
||||
// ── Audio request / response types ───────────────────────────────────────
|
||||
|
||||
// Audio model ids (match backend CommonAudioModel).
|
||||
export type OmniGenAudioModelId =
|
||||
| "suno_music"
|
||||
| "suno_remix"
|
||||
| "suno_sounds"
|
||||
| "suno_sample"
|
||||
| "seed_audio_1p0";
|
||||
|
||||
// Musical keys for audio generation (eg. Suno Sounds).
|
||||
// NB: There are intentionally no E keys, per product spec.
|
||||
export type OmniGenMusicalKey =
|
||||
| "auto"
|
||||
| "c_major"
|
||||
| "c_minor"
|
||||
| "d_major"
|
||||
| "d_minor"
|
||||
| "f_major"
|
||||
| "f_minor"
|
||||
| "g_major"
|
||||
| "g_minor"
|
||||
| "a_major"
|
||||
| "a_minor"
|
||||
| "b_major"
|
||||
| "b_minor";
|
||||
|
||||
/** Shared request body for both the audio cost estimate and audio
|
||||
* generation endpoints. */
|
||||
export interface OmniGenAudioRequest {
|
||||
// REQUIRED (even if marked optional) — prevents duplicate requests.
|
||||
idempotency_token?: string | null;
|
||||
// REQUIRED (even if marked optional) — which model to use.
|
||||
model?: OmniGenAudioModelId | string | null;
|
||||
prompt?: string | null;
|
||||
// Style/genre direction (Suno's "tags").
|
||||
style_prompt?: string | null;
|
||||
// The remix/sample source, or Seed Audio @Audio references (up to 3).
|
||||
audio_media_tokens?: string[] | null;
|
||||
// Seed Audio supports a single reference image; cannot be combined with
|
||||
// audio references.
|
||||
image_media_tokens?: string[] | null;
|
||||
// Whether to keep the original lyrics (Suno Remix).
|
||||
keep_lyrics?: boolean | null;
|
||||
// Whether to generate instrumental-only audio (Suno Music / Sample).
|
||||
is_instrumental?: boolean | null;
|
||||
// Whether the sound should loop vs a single hit (Suno Sounds).
|
||||
is_loopable?: boolean | null;
|
||||
// Beats per minute (Suno Sounds).
|
||||
bpm?: number | null;
|
||||
// The musical key to use (Suno Sounds).
|
||||
musical_key?: OmniGenMusicalKey | null;
|
||||
// Output sample rate in Hz (Seed Audio: 8000/16000/24000/32000/44100/48000).
|
||||
sample_rate_hz?: number | null;
|
||||
// Playback speed multiplier (Seed Audio: 0.5–2.0).
|
||||
speed?: number | null;
|
||||
// Volume multiplier (Seed Audio: 0.5–2.0).
|
||||
volume?: number | null;
|
||||
// Pitch shift in semitones (Seed Audio: -12..=12).
|
||||
pitch?: number | null;
|
||||
}
|
||||
|
||||
export interface OmniGenAudioCostResponse {
|
||||
success: boolean;
|
||||
cost_in_credits?: number | null;
|
||||
cost_in_usd_cents?: number | null;
|
||||
// Whether failures are refunded. True: 100% yes. False: 100% no.
|
||||
// Null/undefined: unknown or variable.
|
||||
failures_are_refunded?: boolean | null;
|
||||
has_watermark: boolean;
|
||||
is_free: boolean;
|
||||
is_rate_limited: boolean;
|
||||
is_unlimited: boolean;
|
||||
}
|
||||
|
||||
export interface OmniGenAudioGenerateResponse {
|
||||
success: boolean;
|
||||
inference_job_token: string;
|
||||
// All job tokens created by this request (including the primary). One
|
||||
// request may create multiple jobs (eg. Suno multi-clip) — poll them all.
|
||||
all_job_tokens: string[];
|
||||
}
|
||||
|
||||
// ── Image model info (from GET /v1/omni_gen/models/image) ────────────────
|
||||
|
||||
export interface OmniGenImageModelInfo {
|
||||
@@ -168,6 +251,47 @@ export interface OmniGenVideoModelsResponse {
|
||||
providers: OmniGenProviderEntry[];
|
||||
}
|
||||
|
||||
// ── Audio model info (from GET /v1/omni_gen/models/audio) ────────────────
|
||||
|
||||
// Unlike the image/video model infos, absent capabilities are omitted from
|
||||
// the JSON entirely (serde skip_serializing_if), so every field except
|
||||
// `model` is optional. Treat undefined as unsupported.
|
||||
export interface OmniGenAudioModelDetails {
|
||||
model: OmniGenAudioModelId | string;
|
||||
// ModelCreator enum as a snake_case string, e.g. "suno", "bytedance".
|
||||
model_creator?: string | null;
|
||||
full_name?: string | null;
|
||||
extra_info?: string | null;
|
||||
extra_info_short?: string | null;
|
||||
text_prompt_supported?: boolean | null;
|
||||
// Whether a style/genre prompt (Suno's "tags") is supported.
|
||||
style_prompt_supported?: boolean | null;
|
||||
audio_references_supported?: boolean | null;
|
||||
audio_references_max?: number | null;
|
||||
image_references_supported?: boolean | null;
|
||||
image_references_max?: number | null;
|
||||
// Whether the "keep lyrics" toggle is supported (Suno Remix).
|
||||
keep_lyrics_supported?: boolean | null;
|
||||
// Whether the instrumental-only toggle is supported (Suno Music / Sample).
|
||||
instrumental_toggle_supported?: boolean | null;
|
||||
// Whether the loop vs single-hit toggle is supported (Suno Sounds).
|
||||
loopable_toggle_supported?: boolean | null;
|
||||
bpm_supported?: boolean | null;
|
||||
musical_key_supported?: boolean | null;
|
||||
sample_rate_hz_options?: number[] | null;
|
||||
sample_rate_hz_default?: number | null;
|
||||
speed_supported?: boolean | null;
|
||||
volume_supported?: boolean | null;
|
||||
pitch_supported?: boolean | null;
|
||||
is_disabled?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OmniGenAudioModelsResponse {
|
||||
success: boolean;
|
||||
models: OmniGenAudioModelDetails[];
|
||||
providers: OmniGenProviderEntry[];
|
||||
}
|
||||
|
||||
// ── Provider types (shared by image and video model responses) ───────────
|
||||
|
||||
export interface OmniGenProviderModelEntry {
|
||||
@@ -213,6 +337,16 @@ export class OmniGenApi extends ApiManager {
|
||||
});
|
||||
}
|
||||
|
||||
public async getAudioModels(
|
||||
provider?: string,
|
||||
): Promise<OmniGenAudioModelsResponse> {
|
||||
const query = provider ? { provider } : undefined;
|
||||
return this.get<OmniGenAudioModelsResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/models/audio`,
|
||||
query,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Cost estimates ───────────────────────────────────────────────────
|
||||
|
||||
public async estimateImageCost(
|
||||
@@ -233,6 +367,15 @@ export class OmniGenApi extends ApiManager {
|
||||
});
|
||||
}
|
||||
|
||||
public async estimateAudioCost(
|
||||
body: OmniGenAudioRequest,
|
||||
): Promise<OmniGenAudioCostResponse> {
|
||||
return this.post<Record<string, unknown>, OmniGenAudioCostResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/cost/audio`,
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Generation ───────────────────────────────────────────────────────
|
||||
|
||||
public async generateImage(
|
||||
@@ -252,4 +395,13 @@ export class OmniGenApi extends ApiManager {
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
|
||||
public async generateAudio(
|
||||
body: OmniGenAudioRequest,
|
||||
): Promise<OmniGenAudioGenerateResponse> {
|
||||
return this.post<Record<string, unknown>, OmniGenAudioGenerateResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/generate/audio`,
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,12 @@ export function getMediaThumbnail(
|
||||
|
||||
let thumbnailUrl: string | null = null;
|
||||
|
||||
if (mediaClass === "audio") {
|
||||
// Audio has no image thumbnail; falling through to cdn_url would hand an
|
||||
// mp3 to an <img>. Callers render a placeholder/waveform instead.
|
||||
return null;
|
||||
}
|
||||
|
||||
if (mediaClass === "video" && mediaLinks.maybe_video_previews?.animated) {
|
||||
thumbnailUrl = mediaLinks.maybe_video_previews.animated;
|
||||
} else if (mediaLinks.maybe_thumbnail_template) {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "@storyteller/ui-audio-player",
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"development": "./src/index.ts",
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"wavesurfer.js": "^7.7.11"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from "./lib/WaveformAudioPlayer.js";
|
||||
export * from "./lib/audio-playback-controller.js";
|
||||
@@ -0,0 +1,369 @@
|
||||
import {
|
||||
MouseEvent,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useId,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faPause,
|
||||
faPlay,
|
||||
faVolume,
|
||||
faVolumeSlash,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import WaveSurfer from "wavesurfer.js";
|
||||
import { addCorsParam } from "@storyteller/common";
|
||||
import { SliderV2 } from "@storyteller/ui-sliderv2";
|
||||
import {
|
||||
notifyAudioStopped,
|
||||
registerAudioPlayer,
|
||||
requestAudioPlayback,
|
||||
unregisterAudioPlayer,
|
||||
} from "./audio-playback-controller.js";
|
||||
|
||||
const WAVE_COLOR = "rgba(255, 255, 255, 0.35)";
|
||||
// Tailwind `primary` (#2d81ff) — canvas fillStyle can't resolve CSS vars, so
|
||||
// the hex is inlined. Keep in sync with the apps' tailwind primary color.
|
||||
const WAVE_PROGRESS_COLOR = "#2d81ff";
|
||||
const PLACEHOLDER_BAR_COUNT = 48;
|
||||
|
||||
// Decoded waveforms are expensive (full fetch + AudioContext decode), so we
|
||||
// keep them per URL for the lifetime of the session. Re-mounts (view toggles,
|
||||
// lightbox, scroll-recycling) reuse the peaks instead of re-downloading.
|
||||
const peaksCache = new Map<string, { peaks: number[][]; duration: number }>();
|
||||
|
||||
export interface WaveformAudioPlayerProps {
|
||||
src: string;
|
||||
// Known duration (e.g. MediaFile.maybe_duration_millis) shown before the
|
||||
// audio metadata loads.
|
||||
durationMillis?: number | null;
|
||||
// Tighter layout for list rows.
|
||||
compact?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function WaveformAudioPlayer({
|
||||
src,
|
||||
durationMillis,
|
||||
compact = false,
|
||||
className = "",
|
||||
}: WaveformAudioPlayerProps) {
|
||||
const playerId = useId();
|
||||
const waveContainerRef = useRef<HTMLDivElement | null>(null);
|
||||
const rootRef = useRef<HTMLDivElement | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const waveSurferRef = useRef<WaveSurfer | null>(null);
|
||||
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
const [duration, setDuration] = useState(
|
||||
durationMillis ? durationMillis / 1000 : 0,
|
||||
);
|
||||
// "pending" until the card scrolls into view, then "ready" once wavesurfer
|
||||
// has drawn, or "failed" if fetch/decode errored (CORS etc.) — playback
|
||||
// still works through the bare <audio> element in that case.
|
||||
const [waveState, setWaveState] = useState<"pending" | "ready" | "failed">(
|
||||
"pending",
|
||||
);
|
||||
const [volume, setVolume] = useState(100);
|
||||
const [showVolume, setShowVolume] = useState(false);
|
||||
|
||||
const waveHeight = compact ? 32 : 44;
|
||||
|
||||
// ── Audio element (owns playback in both wave and fallback modes) ──────
|
||||
useEffect(() => {
|
||||
const audio = new Audio();
|
||||
audio.preload = "metadata";
|
||||
audio.src = src;
|
||||
audioRef.current = audio;
|
||||
|
||||
const onPlay = () => {
|
||||
setIsPlaying(true);
|
||||
requestAudioPlayback(playerId);
|
||||
};
|
||||
const onPause = () => {
|
||||
setIsPlaying(false);
|
||||
notifyAudioStopped(playerId);
|
||||
};
|
||||
const onTimeUpdate = () => setCurrentTime(audio.currentTime);
|
||||
const onLoadedMetadata = () => {
|
||||
if (Number.isFinite(audio.duration) && audio.duration > 0) {
|
||||
setDuration(audio.duration);
|
||||
}
|
||||
};
|
||||
|
||||
audio.addEventListener("play", onPlay);
|
||||
audio.addEventListener("pause", onPause);
|
||||
audio.addEventListener("ended", onPause);
|
||||
audio.addEventListener("timeupdate", onTimeUpdate);
|
||||
audio.addEventListener("loadedmetadata", onLoadedMetadata);
|
||||
|
||||
registerAudioPlayer(playerId, () => audio.pause());
|
||||
|
||||
return () => {
|
||||
unregisterAudioPlayer(playerId);
|
||||
audio.pause();
|
||||
audio.removeEventListener("play", onPlay);
|
||||
audio.removeEventListener("pause", onPause);
|
||||
audio.removeEventListener("ended", onPause);
|
||||
audio.removeEventListener("timeupdate", onTimeUpdate);
|
||||
audio.removeEventListener("loadedmetadata", onLoadedMetadata);
|
||||
audio.removeAttribute("src");
|
||||
audio.load();
|
||||
audioRef.current = null;
|
||||
};
|
||||
}, [src, playerId]);
|
||||
|
||||
// ── Wavesurfer visualization (lazy: only once scrolled into view) ──────
|
||||
useEffect(() => {
|
||||
const root = rootRef.current;
|
||||
if (!root) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const initWaveSurfer = () => {
|
||||
const container = waveContainerRef.current;
|
||||
const media = audioRef.current;
|
||||
if (cancelled || !container || !media || waveSurferRef.current) return;
|
||||
|
||||
const cached = peaksCache.get(src);
|
||||
try {
|
||||
const ws = WaveSurfer.create({
|
||||
container,
|
||||
media,
|
||||
// The waveform decode goes through fetch(), which needs CORS
|
||||
// headers from the CDN; playback (the media element) does not.
|
||||
url: addCorsParam(src) ?? src,
|
||||
height: waveHeight,
|
||||
waveColor: WAVE_COLOR,
|
||||
progressColor: WAVE_PROGRESS_COLOR,
|
||||
cursorColor: "transparent",
|
||||
barWidth: 2,
|
||||
barGap: 1.5,
|
||||
barRadius: 2,
|
||||
normalize: true,
|
||||
interact: true,
|
||||
...(cached
|
||||
? { peaks: cached.peaks, duration: cached.duration }
|
||||
: {}),
|
||||
});
|
||||
waveSurferRef.current = ws;
|
||||
|
||||
ws.on("decode", (decodedDuration: number) => {
|
||||
if (cancelled) return;
|
||||
setWaveState("ready");
|
||||
if (decodedDuration > 0) setDuration(decodedDuration);
|
||||
if (!peaksCache.has(src)) {
|
||||
try {
|
||||
peaksCache.set(src, {
|
||||
peaks: ws.exportPeaks({ maxLength: 1024 }),
|
||||
duration: decodedDuration,
|
||||
});
|
||||
} catch {
|
||||
// Peaks export is a cache optimization only — safe to skip.
|
||||
}
|
||||
}
|
||||
});
|
||||
ws.on("error", () => {
|
||||
if (cancelled) return;
|
||||
setWaveState("failed");
|
||||
waveSurferRef.current = null;
|
||||
try {
|
||||
ws.destroy();
|
||||
} catch {
|
||||
// Already torn down.
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
if (!cancelled) setWaveState("failed");
|
||||
}
|
||||
};
|
||||
|
||||
if (typeof IntersectionObserver === "undefined") {
|
||||
initWaveSurfer();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
destroyWaveSurfer(waveSurferRef);
|
||||
};
|
||||
}
|
||||
|
||||
const observer = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (entries.some((entry) => entry.isIntersecting)) {
|
||||
observer.disconnect();
|
||||
initWaveSurfer();
|
||||
}
|
||||
},
|
||||
{ threshold: 0.15 },
|
||||
);
|
||||
observer.observe(root);
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
observer.disconnect();
|
||||
destroyWaveSurfer(waveSurferRef);
|
||||
};
|
||||
}, [src, waveHeight]);
|
||||
|
||||
// ── Handlers ────────────────────────────────────────────────────────────
|
||||
const togglePlay = useCallback(() => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return;
|
||||
if (audio.paused) {
|
||||
void audio.play().catch(() => setIsPlaying(false));
|
||||
} else {
|
||||
audio.pause();
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleVolumeChange = useCallback((value: number) => {
|
||||
setVolume(value);
|
||||
const audio = audioRef.current;
|
||||
if (audio) audio.volume = value / 100;
|
||||
}, []);
|
||||
|
||||
// Seek on the placeholder / fallback progress bar (the real waveform has
|
||||
// built-in click-to-seek via wavesurfer).
|
||||
const handleBarSeek = useCallback(
|
||||
(event: MouseEvent<HTMLDivElement>) => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio || !duration) return;
|
||||
const rect = event.currentTarget.getBoundingClientRect();
|
||||
const ratio = (event.clientX - rect.left) / rect.width;
|
||||
audio.currentTime = Math.max(0, Math.min(1, ratio)) * duration;
|
||||
},
|
||||
[duration],
|
||||
);
|
||||
|
||||
// Close the volume flyout on outside clicks.
|
||||
useEffect(() => {
|
||||
if (!showVolume) return;
|
||||
const onDocClick = (event: Event) => {
|
||||
if (!rootRef.current?.contains(event.target as Node)) {
|
||||
setShowVolume(false);
|
||||
}
|
||||
};
|
||||
document.addEventListener("mousedown", onDocClick);
|
||||
return () => document.removeEventListener("mousedown", onDocClick);
|
||||
}, [showVolume]);
|
||||
|
||||
const placeholderBars = useMemo(() => buildPlaceholderBars(src), [src]);
|
||||
const progressRatio = duration > 0 ? currentTime / duration : 0;
|
||||
const showWave = waveState !== "failed";
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={rootRef}
|
||||
className={`flex items-center gap-2.5 ${compact ? "px-2.5 py-2" : "px-3 py-2.5"} ${className}`}
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
aria-label={isPlaying ? "Pause" : "Play"}
|
||||
onClick={togglePlay}
|
||||
className={`flex shrink-0 items-center justify-center rounded-full bg-white text-black transition-transform hover:scale-105 active:scale-95 ${compact ? "h-8 w-8" : "h-10 w-10"}`}
|
||||
>
|
||||
<FontAwesomeIcon
|
||||
icon={isPlaying ? faPause : faPlay}
|
||||
className={`${compact ? "h-3 w-3" : "h-3.5 w-3.5"} ${isPlaying ? "" : "ml-0.5"}`}
|
||||
/>
|
||||
</button>
|
||||
|
||||
<div className="relative min-w-0 flex-1" style={{ height: waveHeight }}>
|
||||
{showWave && (
|
||||
<div ref={waveContainerRef} className="absolute inset-0" />
|
||||
)}
|
||||
{(waveState === "pending" || waveState === "failed") && (
|
||||
<div
|
||||
className="absolute inset-0 flex cursor-pointer items-center gap-[1.5px]"
|
||||
onClick={handleBarSeek}
|
||||
>
|
||||
{waveState === "failed" ? (
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-white/20">
|
||||
<div
|
||||
className="h-full rounded-full bg-primary"
|
||||
style={{ width: `${progressRatio * 100}%` }}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
placeholderBars.map((barHeight, index) => (
|
||||
<div
|
||||
key={index}
|
||||
className="w-[2px] shrink-0 rounded-full bg-white/25"
|
||||
style={{ height: `${barHeight}%` }}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<span
|
||||
className={`shrink-0 tabular-nums text-white/70 ${compact ? "text-[11px]" : "text-xs"}`}
|
||||
>
|
||||
{formatTime(currentTime)} / {duration > 0 ? formatTime(duration) : "-:--"}
|
||||
</span>
|
||||
|
||||
<div className="relative shrink-0">
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Volume"
|
||||
onClick={() => setShowVolume((visible) => !visible)}
|
||||
className="flex h-7 w-7 items-center justify-center rounded-md text-white/70 transition-colors hover:bg-white/10 hover:text-white"
|
||||
>
|
||||
<FontAwesomeIcon
|
||||
icon={volume === 0 ? faVolumeSlash : faVolume}
|
||||
className="h-3.5 w-3.5"
|
||||
/>
|
||||
</button>
|
||||
{showVolume && (
|
||||
<div className="absolute bottom-full right-0 z-20 mb-2 w-28 rounded-lg border border-white/10 bg-neutral-900/95 px-3 py-2.5 shadow-xl backdrop-blur">
|
||||
<SliderV2
|
||||
min={0}
|
||||
max={100}
|
||||
step={1}
|
||||
value={volume}
|
||||
onChange={handleVolumeChange}
|
||||
variant="filled"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function destroyWaveSurfer(ref: { current: WaveSurfer | null }): void {
|
||||
if (!ref.current) return;
|
||||
try {
|
||||
ref.current.destroy();
|
||||
} catch {
|
||||
// Already torn down.
|
||||
}
|
||||
ref.current = null;
|
||||
}
|
||||
|
||||
function formatTime(totalSeconds: number): string {
|
||||
const seconds = Math.max(0, Math.floor(totalSeconds));
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainder = seconds % 60;
|
||||
return `${minutes}:${remainder.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
// Deterministic pseudo-waveform so unloaded cards still read as audio and
|
||||
// don't all look identical. Seeded by the URL, not random — stable renders.
|
||||
function buildPlaceholderBars(seedSource: string): number[] {
|
||||
let seed = 0;
|
||||
for (let index = 0; index < seedSource.length; index++) {
|
||||
seed = (seed * 31 + seedSource.charCodeAt(index)) >>> 0;
|
||||
}
|
||||
const bars: number[] = [];
|
||||
for (let index = 0; index < PLACEHOLDER_BAR_COUNT; index++) {
|
||||
seed = (seed * 1103515245 + 12345) >>> 0;
|
||||
bars.push(25 + (seed % 65));
|
||||
}
|
||||
return bars;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
notifyAudioStopped,
|
||||
registerAudioPlayer,
|
||||
requestAudioPlayback,
|
||||
resetAudioPlaybackControllerForTests,
|
||||
unregisterAudioPlayer,
|
||||
} from "./audio-playback-controller";
|
||||
|
||||
describe("audio-playback-controller", () => {
|
||||
beforeEach(() => {
|
||||
resetAudioPlaybackControllerForTests();
|
||||
});
|
||||
|
||||
it("pauses the previously playing player when another starts", () => {
|
||||
const pauseA = vi.fn();
|
||||
const pauseB = vi.fn();
|
||||
registerAudioPlayer("a", pauseA);
|
||||
registerAudioPlayer("b", pauseB);
|
||||
|
||||
requestAudioPlayback("a");
|
||||
expect(pauseA).not.toHaveBeenCalled();
|
||||
|
||||
requestAudioPlayback("b");
|
||||
expect(pauseA).toHaveBeenCalledTimes(1);
|
||||
expect(pauseB).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not pause a player that re-requests playback (seek/resume)", () => {
|
||||
const pauseA = vi.fn();
|
||||
registerAudioPlayer("a", pauseA);
|
||||
|
||||
requestAudioPlayback("a");
|
||||
requestAudioPlayback("a");
|
||||
expect(pauseA).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clears the current player when it stops on its own", () => {
|
||||
const pauseA = vi.fn();
|
||||
registerAudioPlayer("a", pauseA);
|
||||
|
||||
requestAudioPlayback("a");
|
||||
notifyAudioStopped("a");
|
||||
|
||||
// A new player starting should not pause the already-stopped one.
|
||||
const pauseB = vi.fn();
|
||||
registerAudioPlayer("b", pauseB);
|
||||
requestAudioPlayback("b");
|
||||
expect(pauseA).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("unregistering the current player forgets it", () => {
|
||||
const pauseA = vi.fn();
|
||||
registerAudioPlayer("a", pauseA);
|
||||
requestAudioPlayback("a");
|
||||
|
||||
unregisterAudioPlayer("a");
|
||||
requestAudioPlayback("b");
|
||||
expect(pauseA).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
// Global playback coordinator: ensures only one audio player is audible at a
|
||||
// time. Players register a pause callback; when one starts playing it calls
|
||||
// requestAudioPlayback, which pauses whichever player was playing before it.
|
||||
|
||||
type PauseFn = () => void;
|
||||
|
||||
let currentId: string | null = null;
|
||||
const players = new Map<string, PauseFn>();
|
||||
|
||||
export function registerAudioPlayer(id: string, pause: PauseFn): void {
|
||||
players.set(id, pause);
|
||||
}
|
||||
|
||||
export function unregisterAudioPlayer(id: string): void {
|
||||
players.delete(id);
|
||||
if (currentId === id) {
|
||||
currentId = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Call when a player starts playing. Pauses the previously playing player. */
|
||||
export function requestAudioPlayback(id: string): void {
|
||||
if (currentId && currentId !== id) {
|
||||
players.get(currentId)?.();
|
||||
}
|
||||
currentId = id;
|
||||
}
|
||||
|
||||
/** Call when a player stops on its own (pause/finish/unmount). */
|
||||
export function notifyAudioStopped(id: string): void {
|
||||
if (currentId === id) {
|
||||
currentId = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Test helper: reset module state between test cases. */
|
||||
export function resetAudioPlaybackControllerForTests(): void {
|
||||
currentId = null;
|
||||
players.clear();
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"files": [],
|
||||
"include": [],
|
||||
"references": [
|
||||
{
|
||||
"path": "../slider-v2"
|
||||
},
|
||||
{
|
||||
"path": "../../common"
|
||||
},
|
||||
{
|
||||
"path": "./tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "./tsconfig.spec.json"
|
||||
}
|
||||
],
|
||||
"extends": "../../../tsconfig.base.json"
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"extends": "../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"types": [
|
||||
"node",
|
||||
"@nx/react/typings/cssmodule.d.ts",
|
||||
"@nx/react/typings/image.d.ts",
|
||||
"vite/client"
|
||||
],
|
||||
"rootDir": "src",
|
||||
"jsx": "react-jsx",
|
||||
"tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo"
|
||||
},
|
||||
"exclude": [
|
||||
"out-tsc",
|
||||
"dist",
|
||||
"**/*.spec.ts",
|
||||
"**/*.test.ts",
|
||||
"**/*.spec.tsx",
|
||||
"**/*.test.tsx",
|
||||
"**/*.spec.js",
|
||||
"**/*.test.js",
|
||||
"**/*.spec.jsx",
|
||||
"**/*.test.jsx",
|
||||
"vite.config.ts",
|
||||
"vite.config.mts",
|
||||
"vitest.config.ts",
|
||||
"vitest.config.mts",
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.spec.ts",
|
||||
"src/**/*.test.tsx",
|
||||
"src/**/*.spec.tsx",
|
||||
"src/**/*.test.js",
|
||||
"src/**/*.spec.js",
|
||||
"src/**/*.test.jsx",
|
||||
"src/**/*.spec.jsx"
|
||||
],
|
||||
"include": [
|
||||
"src/**/*.js",
|
||||
"src/**/*.jsx",
|
||||
"src/**/*.ts",
|
||||
"src/**/*.tsx"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"path": "../slider-v2/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../common/tsconfig.lib.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"extends": "../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./out-tsc/vitest",
|
||||
"types": [
|
||||
"vitest/globals",
|
||||
"vitest/importMeta",
|
||||
"vite/client",
|
||||
"node",
|
||||
"vitest"
|
||||
],
|
||||
"jsx": "react-jsx"
|
||||
},
|
||||
"include": [
|
||||
"vite.config.ts",
|
||||
"vite.config.mts",
|
||||
"vitest.config.ts",
|
||||
"vitest.config.mts",
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.spec.ts",
|
||||
"src/**/*.test.tsx",
|
||||
"src/**/*.spec.tsx",
|
||||
"src/**/*.test.js",
|
||||
"src/**/*.spec.js",
|
||||
"src/**/*.test.jsx",
|
||||
"src/**/*.spec.jsx",
|
||||
"src/**/*.d.ts"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"path": "./tsconfig.lib.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/// <reference types='vitest' />
|
||||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
import dts from 'vite-plugin-dts';
|
||||
import * as path from 'path';
|
||||
import { isExternal } from '../../shared-vite-config';
|
||||
|
||||
export default defineConfig(() => ({
|
||||
root: __dirname,
|
||||
cacheDir: '../../../node_modules/.vite/libs/components/audio-player',
|
||||
plugins: [react(), dts({ entryRoot: 'src', tsconfigPath: path.join(__dirname, 'tsconfig.lib.json') })],
|
||||
build: {
|
||||
outDir: './dist',
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
commonjsOptions: {
|
||||
transformMixedEsModules: true,
|
||||
},
|
||||
lib: {
|
||||
entry: 'src/index.ts',
|
||||
name: '@storyteller/ui-audio-player',
|
||||
fileName: 'index',
|
||||
formats: ['es' as const]
|
||||
},
|
||||
rollupOptions: {
|
||||
// External packages that should not be bundled into your library.
|
||||
external: isExternal
|
||||
},
|
||||
},
|
||||
test: {
|
||||
watch: false,
|
||||
globals: true,
|
||||
environment: 'jsdom',
|
||||
include: ['{src,tests}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
|
||||
reporters: ['default'],
|
||||
coverage: {
|
||||
reportsDirectory: './test-output/vitest/coverage',
|
||||
provider: 'v8' as const,
|
||||
}
|
||||
},
|
||||
}));
|
||||
@@ -26,8 +26,10 @@ export const ToggleButton = ({
|
||||
return (
|
||||
<Button
|
||||
className={twMerge(
|
||||
"flex h-9 items-center justify-center rounded-lg border-2 border-transparent text-sm text-white backdrop-blur-lg transition-all",
|
||||
hasLabel ? "px-3" : "w-9",
|
||||
// 34px matches the sibling toolbar controls (GenerateButton, the
|
||||
// PopoverMenu triggers); the 2px border is inside the box.
|
||||
"flex h-[34px] items-center justify-center rounded-lg border-2 border-transparent text-sm text-white backdrop-blur-lg transition-all",
|
||||
hasLabel ? "px-3" : "w-[34px]",
|
||||
isActive
|
||||
? "border-white/20 bg-brand-primary/40 hover:border-white/30 hover:bg-brand-primary/40"
|
||||
: "bg-[#5F5F68]/60 hover:bg-[#5F5F68]/90",
|
||||
|
||||
@@ -7,6 +7,7 @@ export * from "./lib/gallery-view-store";
|
||||
export * from "./lib/GalleryViewToggle";
|
||||
export * from "./lib/useGalleryData";
|
||||
export * from "./lib/useGalleryEntries";
|
||||
export * from "./lib/useGenerationJobs";
|
||||
export * from "./lib/GalleryThumbnail";
|
||||
export * from "./lib/CopyPromptButton";
|
||||
export * from "./lib/PendingRow";
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
import { memo, useCallback, useState, type ReactNode } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faCube, faImage, faVideo } from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
faCube,
|
||||
faImage,
|
||||
faMusic,
|
||||
faVideo,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
getCreatorIconPathForModelId,
|
||||
getModelDisplayName,
|
||||
} from "@storyteller/model-list";
|
||||
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
|
||||
import { GalleryThumbnail } from "./GalleryThumbnail";
|
||||
import type { GalleryItem } from "./types";
|
||||
|
||||
@@ -15,6 +21,9 @@ const STORAGE_KEY = "gallery-aspect-ratios";
|
||||
// Cap ratio so tall portraits don't dominate — 1.4 ≈ 5:7
|
||||
const MAX_RATIO = 1.4;
|
||||
|
||||
// Audio cards have no image to measure — fixed square tile.
|
||||
const AUDIO_RATIO = 1;
|
||||
|
||||
function loadCache(): Map<string, number> {
|
||||
const map = new Map<string, number>();
|
||||
try {
|
||||
@@ -59,6 +68,9 @@ export interface GalleryCardProps {
|
||||
// "auto" = dynamic aspect ratio from the loaded image (masonry layouts).
|
||||
// "square" = fixed 1:1; skips the ratio measurement path (uniform grids).
|
||||
shape?: "auto" | "square";
|
||||
/** Prompt text resolved by the view (via the prompts cache). Audio cards
|
||||
* show it inline since they have no image to speak for themselves. */
|
||||
title?: string;
|
||||
/** Hover-revealed quick-action cluster (recreate / share / download …). */
|
||||
actionsSlot?: ReactNode;
|
||||
}
|
||||
@@ -67,6 +79,7 @@ export const GalleryCard = memo(function GalleryCard({
|
||||
item,
|
||||
onClick,
|
||||
shape = "auto",
|
||||
title,
|
||||
actionsSlot,
|
||||
}: GalleryCardProps) {
|
||||
const isSquare = shape === "square";
|
||||
@@ -75,8 +88,15 @@ export const GalleryCard = memo(function GalleryCard({
|
||||
|
||||
const isVideo = item.mediaClass === "video";
|
||||
const is3D = item.mediaClass === "dimensional";
|
||||
const mediaIcon = isVideo ? faVideo : is3D ? faCube : faImage;
|
||||
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : "Image";
|
||||
const isAudio = item.mediaClass === "audio";
|
||||
const mediaIcon = isVideo
|
||||
? faVideo
|
||||
: is3D
|
||||
? faCube
|
||||
: isAudio
|
||||
? faMusic
|
||||
: faImage;
|
||||
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : isAudio ? "Audio" : "Image";
|
||||
const modelDisplayName = item.modelId
|
||||
? getModelDisplayName(item.modelId)
|
||||
: null;
|
||||
@@ -84,7 +104,11 @@ export const GalleryCard = memo(function GalleryCard({
|
||||
? getCreatorIconPathForModelId(item.modelId)
|
||||
: null;
|
||||
|
||||
const displayRatio = ratio ? Math.min(ratio, MAX_RATIO) : 1;
|
||||
const displayRatio = isAudio
|
||||
? AUDIO_RATIO
|
||||
: ratio
|
||||
? Math.min(ratio, MAX_RATIO)
|
||||
: 1;
|
||||
|
||||
// In square mode the wrapper sets the ratio via `aspect-square`; we only
|
||||
// compute the dynamic aspectRatio for masonry-style layouts.
|
||||
@@ -136,13 +160,42 @@ export const GalleryCard = memo(function GalleryCard({
|
||||
className="absolute inset-0 overflow-hidden rounded-[inherit]"
|
||||
style={{ contentVisibility: "auto", containIntrinsicSize: "auto 200px" }}
|
||||
>
|
||||
<GalleryThumbnail
|
||||
thumbnail={item.thumbnail}
|
||||
alt={item.label}
|
||||
isVideo={isVideo}
|
||||
fallbackIcon={mediaIcon}
|
||||
onLoad={measureRatio}
|
||||
/>
|
||||
{isAudio ? (
|
||||
<div className="flex h-full flex-col bg-gradient-to-br from-white/[0.07] to-white/[0.02] p-3">
|
||||
<div className="flex items-start gap-2">
|
||||
<p className="line-clamp-3 min-w-0 flex-1 text-sm leading-snug text-white/85">
|
||||
{title || item.label}
|
||||
</p>
|
||||
<FontAwesomeIcon
|
||||
icon={faMusic}
|
||||
className="pointer-events-none mt-0.5 shrink-0 text-base text-white/15"
|
||||
/>
|
||||
</div>
|
||||
{item.fullImage ? (
|
||||
<div className="flex min-h-0 flex-1 items-center">
|
||||
<div className="w-full">
|
||||
<WaveformAudioPlayer
|
||||
src={item.fullImage}
|
||||
durationMillis={item.durationMillis}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<FontAwesomeIcon
|
||||
icon={faMusic}
|
||||
className="m-auto text-2xl text-white/20"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<GalleryThumbnail
|
||||
thumbnail={item.thumbnail}
|
||||
alt={item.label}
|
||||
isVideo={isVideo}
|
||||
fallbackIcon={mediaIcon}
|
||||
onLoad={measureRatio}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Hover overlay with media type + model badges and quick actions */}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { memo, useCallback, type ReactNode } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faCube, faImage, faPlay, faVideo } from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
faCube,
|
||||
faImage,
|
||||
faMusic,
|
||||
faPlay,
|
||||
faVideo,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
getCreatorIconPathForModelId,
|
||||
getModelDisplayName,
|
||||
} from "@storyteller/model-list";
|
||||
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
|
||||
import { GalleryThumbnail } from "./GalleryThumbnail";
|
||||
import { CopyPromptButton } from "./CopyPromptButton";
|
||||
import { formatTimeAgo } from "./format-time-ago";
|
||||
@@ -35,8 +42,15 @@ export const GalleryRow = memo(function GalleryRow({
|
||||
}: GalleryRowProps) {
|
||||
const isVideo = item.mediaClass === "video";
|
||||
const is3D = item.mediaClass === "dimensional";
|
||||
const mediaIcon = isVideo ? faVideo : is3D ? faCube : faImage;
|
||||
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : "Image";
|
||||
const isAudio = item.mediaClass === "audio";
|
||||
const mediaIcon = isVideo
|
||||
? faVideo
|
||||
: is3D
|
||||
? faCube
|
||||
: isAudio
|
||||
? faMusic
|
||||
: faImage;
|
||||
const mediaLabel = isVideo ? "Video" : is3D ? "3D" : isAudio ? "Audio" : "Image";
|
||||
|
||||
const effectiveModelId = modelId ?? item.modelId;
|
||||
const modelDisplayName = effectiveModelId
|
||||
@@ -108,6 +122,15 @@ export const GalleryRow = memo(function GalleryRow({
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{isAudio && item.fullImage && (
|
||||
<div className="mt-1.5 max-w-md rounded-md bg-white/[0.04]">
|
||||
<WaveformAudioPlayer
|
||||
src={item.fullImage}
|
||||
durationMillis={item.durationMillis}
|
||||
compact
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
<div className="mt-1 flex flex-wrap items-center gap-x-1.5 gap-y-1 text-xs text-white/45">
|
||||
{modelIconPath && (
|
||||
<img
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { ReactNode } from "react";
|
||||
import { useCallback, useMemo, type ReactNode } from "react";
|
||||
import Masonry from "react-masonry-css";
|
||||
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
|
||||
import { PendingCard } from "./PendingCard";
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
useMergedGalleryEntries,
|
||||
useInfiniteScrollSentinel,
|
||||
} from "./useGalleryEntries";
|
||||
import { usePrompts } from "./prompts-cache";
|
||||
import { useMediaPromptTokens } from "./media-prompt-token-cache";
|
||||
import type { RecreateSlotContext } from "./GenerationListView";
|
||||
import type { FailedJob, GalleryItem, InProgressJob } from "./types";
|
||||
|
||||
@@ -72,6 +74,51 @@ export function GenerationGridView({
|
||||
newlyCompletedTokens,
|
||||
});
|
||||
|
||||
// Audio cards show their prompt inline (no image speaks for them), so
|
||||
// resolve prompt text the same way the list view does — but only for audio
|
||||
// items, keeping image/video walls free of the extra lookups. Both steps
|
||||
// (media token → prompt token → prompt text) are batched + cached.
|
||||
const audioMediaTokensNeedingPrompt = useMemo(
|
||||
() =>
|
||||
mergedEntries.flatMap((entry) =>
|
||||
entry.kind === "gallery" &&
|
||||
entry.item.mediaClass === "audio" &&
|
||||
!entry.item.promptToken
|
||||
? [entry.item.id]
|
||||
: [],
|
||||
),
|
||||
[mergedEntries],
|
||||
);
|
||||
const resolvedPromptTokens = useMediaPromptTokens(
|
||||
audioMediaTokensNeedingPrompt,
|
||||
);
|
||||
|
||||
const audioPromptTokens = useMemo(() => {
|
||||
const tokens: string[] = [];
|
||||
for (const entry of mergedEntries) {
|
||||
if (entry.kind !== "gallery" || entry.item.mediaClass !== "audio") {
|
||||
continue;
|
||||
}
|
||||
const token =
|
||||
entry.item.promptToken || resolvedPromptTokens.get(entry.item.id);
|
||||
if (token) tokens.push(token);
|
||||
}
|
||||
return tokens;
|
||||
}, [mergedEntries, resolvedPromptTokens]);
|
||||
const promptsMap = usePrompts(audioPromptTokens);
|
||||
|
||||
const audioTitleFor = useCallback(
|
||||
(item: GalleryItem): string | undefined => {
|
||||
if (item.mediaClass !== "audio") return undefined;
|
||||
const token = item.promptToken || resolvedPromptTokens.get(item.id);
|
||||
return (
|
||||
(token ? promptsMap.get(token)?.maybe_positive_prompt : undefined) ||
|
||||
undefined
|
||||
);
|
||||
},
|
||||
[resolvedPromptTokens, promptsMap],
|
||||
);
|
||||
|
||||
if (isInitialLoading) {
|
||||
return (
|
||||
<div className="flex justify-center py-20">
|
||||
@@ -135,6 +182,7 @@ export function GenerationGridView({
|
||||
<GalleryCard
|
||||
item={entry.item}
|
||||
onClick={onGalleryItemClick}
|
||||
title={audioTitleFor(entry.item)}
|
||||
actionsSlot={renderGalleryActions?.(entry.item, {
|
||||
modelId: entry.item.modelId,
|
||||
promptToken: entry.item.promptToken,
|
||||
|
||||
@@ -9,7 +9,12 @@ import {
|
||||
} from "./useGalleryEntries";
|
||||
import { usePrompts } from "./prompts-cache";
|
||||
import { useMediaPromptTokens } from "./media-prompt-token-cache";
|
||||
import type { FailedJob, GalleryItem, InProgressJob } from "./types";
|
||||
import type {
|
||||
FailedJob,
|
||||
GalleryItem,
|
||||
GenerationMediaClass,
|
||||
InProgressJob,
|
||||
} from "./types";
|
||||
|
||||
// Constrains the feed to the promptbox width (max-w-5xl) and stacks one row
|
||||
// per generation: a single time-sorted merge of the in-progress / failed /
|
||||
@@ -19,7 +24,7 @@ import type { FailedJob, GalleryItem, InProgressJob } from "./types";
|
||||
|
||||
export interface RecreateSlotContext {
|
||||
promptToken: string;
|
||||
mediaClass: "image" | "video";
|
||||
mediaClass: GenerationMediaClass;
|
||||
kind: "pending" | "failed";
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faSpinnerThird } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { getCreatorIconPathForModelId } from "@storyteller/model-list";
|
||||
import { derivePendingStatus } from "./pending-status";
|
||||
import { batchNoun } from "./types";
|
||||
import type { GenerationMediaClass } from "./types";
|
||||
|
||||
export interface PendingCardProps {
|
||||
id: string;
|
||||
@@ -12,7 +14,7 @@ export interface PendingCardProps {
|
||||
progress?: number;
|
||||
estimatedTimeLeftMs?: number;
|
||||
batchCount?: number;
|
||||
mediaClass: "image" | "video";
|
||||
mediaClass: GenerationMediaClass;
|
||||
/** Hover-revealed action (e.g. a Recreate button) next to the prompt. */
|
||||
recreateSlot?: ReactNode;
|
||||
}
|
||||
@@ -39,8 +41,7 @@ export const PendingCard = memo(function PendingCard({
|
||||
<div className="animate-shimmer h-full w-full" />
|
||||
{batchCount != null && batchCount > 1 && (
|
||||
<div className="absolute left-2 right-2 top-2 z-10 rounded-md bg-black/60 px-2.5 py-1.5 text-center text-[10px] leading-snug text-white/70 backdrop-blur-sm">
|
||||
Generating {batchCount}{" "}
|
||||
{mediaClass === "image" ? "images" : "videos"} · Results may appear
|
||||
Generating {batchCount} {batchNoun(mediaClass)} · Results may appear
|
||||
one at a time
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -4,6 +4,8 @@ import { faSpinnerThird } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { getCreatorIconPathForModelId } from "@storyteller/model-list";
|
||||
import { CopyPromptButton } from "./CopyPromptButton";
|
||||
import { derivePendingStatus } from "./pending-status";
|
||||
import { batchNoun } from "./types";
|
||||
import type { GenerationMediaClass } from "./types";
|
||||
|
||||
export interface PendingRowProps {
|
||||
id: string;
|
||||
@@ -13,7 +15,7 @@ export interface PendingRowProps {
|
||||
progress?: number;
|
||||
estimatedTimeLeftMs?: number;
|
||||
batchCount?: number;
|
||||
mediaClass: "image" | "video";
|
||||
mediaClass: GenerationMediaClass;
|
||||
/** Hover-revealed action (e.g. a Recreate button) after the model label. */
|
||||
recreateSlot?: ReactNode;
|
||||
onCopyPromptResult?: (success: boolean) => void;
|
||||
@@ -78,7 +80,7 @@ export const PendingRow = memo(function PendingRow({
|
||||
<>
|
||||
<span className="text-white/25">·</span>
|
||||
<span className="shrink-0">
|
||||
{batchCount} {mediaClass === "image" ? "images" : "videos"}
|
||||
{batchCount} {batchNoun(mediaClass)}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -1,6 +1,20 @@
|
||||
// Canonical data shapes for the merged generation feed (in-progress / failed /
|
||||
// completed). Hosts (webapp, desktop) map their own job sources into these.
|
||||
|
||||
export type GenerationMediaClass = "image" | "video" | "audio";
|
||||
|
||||
/** Plural noun for batch captions, e.g. "Generating 4 images". */
|
||||
export function batchNoun(mediaClass: GenerationMediaClass): string {
|
||||
switch (mediaClass) {
|
||||
case "video":
|
||||
return "videos";
|
||||
case "audio":
|
||||
return "audio clips";
|
||||
default:
|
||||
return "images";
|
||||
}
|
||||
}
|
||||
|
||||
export interface GalleryItem {
|
||||
id: string;
|
||||
label: string;
|
||||
@@ -13,6 +27,8 @@ export interface GalleryItem {
|
||||
// Token for the generation's prompt record. The list view resolves it
|
||||
// (via the shared prompts cache) to show the real prompt + model.
|
||||
promptToken?: string;
|
||||
// Playback length for audio (and video) items, when the API knows it.
|
||||
durationMillis?: number;
|
||||
}
|
||||
|
||||
export interface InProgressJob {
|
||||
@@ -27,7 +43,7 @@ export interface InProgressJob {
|
||||
// Prompt token + media class enable the "Recreate" action while the job is
|
||||
// still running, mirroring the failed/completed cards.
|
||||
promptToken?: string;
|
||||
mediaClass: "image" | "video";
|
||||
mediaClass: GenerationMediaClass;
|
||||
}
|
||||
|
||||
export interface FailedJob {
|
||||
@@ -41,5 +57,5 @@ export interface FailedJob {
|
||||
createdAt: string;
|
||||
promptToken?: string;
|
||||
refImageUrl?: string;
|
||||
mediaClass: "image" | "video";
|
||||
mediaClass: GenerationMediaClass;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ const getLabel = (item: any) => {
|
||||
return "Image Generation";
|
||||
case "video":
|
||||
return "Video Generation";
|
||||
case "audio":
|
||||
return "Audio Generation";
|
||||
case "dimensional":
|
||||
return "3D Mesh";
|
||||
default:
|
||||
@@ -42,12 +44,15 @@ export function useGalleryData(options: {
|
||||
const api = useMemo(() => new GalleryModalApi(), []);
|
||||
|
||||
const mapApiItem = useCallback((item: any): GalleryItem => {
|
||||
const isDimensional = item.media_class === "dimensional";
|
||||
const thumbnail = isDimensional
|
||||
? null
|
||||
: getMediaThumbnail(item.media_links, item.media_class, {
|
||||
// 3D and audio media have no image thumbnail — their cards render a
|
||||
// placeholder icon / waveform player instead.
|
||||
const hasImageThumbnail =
|
||||
item.media_class !== "dimensional" && item.media_class !== "audio";
|
||||
const thumbnail = hasImageThumbnail
|
||||
? getMediaThumbnail(item.media_links, item.media_class, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
})
|
||||
: null;
|
||||
|
||||
return {
|
||||
id: item.token,
|
||||
@@ -59,6 +64,7 @@ export function useGalleryData(options: {
|
||||
modelId: item.maybe_model_type || undefined,
|
||||
batchImageToken: item.maybe_batch_token,
|
||||
promptToken: item.maybe_prompt_token || undefined,
|
||||
durationMillis: item.maybe_duration_millis ?? undefined,
|
||||
};
|
||||
}, []);
|
||||
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { JobsApi, JobStatus, MediaFilesApi } from "@storyteller/api";
|
||||
import type { Job, Prompts } from "@storyteller/api";
|
||||
import { getMediaThumbnail, THUMBNAIL_SIZES } from "@storyteller/common";
|
||||
import {
|
||||
getModelDisplayName,
|
||||
getProviderDisplayName,
|
||||
ALL_MODELS_LIST,
|
||||
} from "@storyteller/model-list";
|
||||
import { getCachedPrompt, usePrompts } from "./prompts-cache";
|
||||
import type {
|
||||
FailedJob,
|
||||
GalleryItem,
|
||||
GenerationMediaClass,
|
||||
InProgressJob,
|
||||
} from "./types";
|
||||
|
||||
// Maps JobsApi's recent-jobs stream into the canonical feed shapes
|
||||
// (InProgressJob / FailedJob / newly-completed GalleryItem). HTTP-based, so
|
||||
// both the webapp and the desktop app can use it (the desktop image/video
|
||||
// pages use the Tauri task queue instead; audio enqueues over HTTP).
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────
|
||||
|
||||
const IN_PROGRESS_STATUSES = new Set([JobStatus.PENDING, JobStatus.STARTED]);
|
||||
const COMPLETED_STATUSES = new Set([JobStatus.COMPLETE_SUCCESS]);
|
||||
const FAILED_STATUSES = new Set([
|
||||
JobStatus.ATTEMPT_FAILED,
|
||||
JobStatus.COMPLETE_FAILURE,
|
||||
JobStatus.DEAD,
|
||||
JobStatus.CANCELLED_BY_USER,
|
||||
JobStatus.CANCELLED_BY_SYSTEM,
|
||||
]);
|
||||
|
||||
const FAILURE_REASON_LABEL: Record<string, string> = {
|
||||
rule_bans_user_image: "Image violates content policy",
|
||||
rule_bans_user_image_with_faces: "Images with faces are not allowed",
|
||||
rule_bans_user_text_prompt: "Text prompt violates content policy",
|
||||
rule_bans_user_content: "Content violates content policy",
|
||||
rule_bans_generated_video: "Generated video flagged by content policy",
|
||||
rule_bans_generated_audio: "Generated audio flagged by content policy",
|
||||
rule_bans_generated_content: "Generated content flagged by content policy",
|
||||
generation_failed: "Generation failed",
|
||||
unknown: "An unknown error occurred",
|
||||
};
|
||||
|
||||
// Progress-bar defaults when the model doesn't declare a progressBarTime.
|
||||
// Mirrors the backend's estimate baselines (audio 120s, video 300s+).
|
||||
const DEFAULT_DURATION_MS: Record<GenerationMediaClass, number> = {
|
||||
image: 30000,
|
||||
video: 900000,
|
||||
audio: 120000,
|
||||
};
|
||||
|
||||
// Cache per-task durations
|
||||
const taskDurationCache = new Map<string, number>();
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function getJobMediaType(
|
||||
job: Job,
|
||||
): "image" | "video" | "audio" | "other" {
|
||||
const cat = job.request.inference_category?.toLowerCase() ?? "";
|
||||
if (cat.includes("video")) return "video";
|
||||
if (cat.includes("image")) return "image";
|
||||
if (cat.includes("audio")) return "audio";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function getJobMediaClass(job: Job): GenerationMediaClass {
|
||||
const inferred = getJobMediaType(job);
|
||||
return inferred === "other" ? "image" : inferred;
|
||||
}
|
||||
|
||||
function getModelLabel(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
|
||||
const modelType =
|
||||
cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
|
||||
const providerKey = cachedPrompt?.maybe_generation_provider ?? modelType;
|
||||
|
||||
const modelDisplay = modelType ? getModelDisplayName(modelType) : undefined;
|
||||
const provider = providerKey
|
||||
? getProviderDisplayName(providerKey.toLowerCase())
|
||||
: undefined;
|
||||
|
||||
if (modelDisplay && provider) return `${modelDisplay} · ${provider}`;
|
||||
return modelDisplay || provider || "Unknown model";
|
||||
}
|
||||
|
||||
function getPrompt(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cached = promptToken
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
return (
|
||||
cached?.maybe_positive_prompt || job.request.maybe_raw_inference_text || ""
|
||||
);
|
||||
}
|
||||
|
||||
function getModelId(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
? (promptsMap?.get(promptToken) ?? getCachedPrompt(promptToken))
|
||||
: undefined;
|
||||
return cachedPrompt?.maybe_model_type ?? job.request.maybe_model_type ?? "";
|
||||
}
|
||||
|
||||
function jobToInProgress(
|
||||
job: Job,
|
||||
promptsMap: Map<string, Prompts>,
|
||||
): InProgressJob {
|
||||
const now = Date.now();
|
||||
const createdMs = new Date(job.created_at).getTime();
|
||||
const modelType = job.request.maybe_model_type;
|
||||
const mediaClass = getJobMediaClass(job);
|
||||
|
||||
let duration = taskDurationCache.get(job.job_token);
|
||||
if (!duration) {
|
||||
const model = modelType
|
||||
? ALL_MODELS_LIST.find(
|
||||
(m) => m.tauriId === modelType || m.id === modelType,
|
||||
)
|
||||
: undefined;
|
||||
duration = model?.progressBarTime ?? DEFAULT_DURATION_MS[mediaClass];
|
||||
taskDurationCache.set(job.job_token, duration);
|
||||
}
|
||||
|
||||
const elapsed = now - createdMs;
|
||||
const progress = Math.min(95, (elapsed / duration) * 100);
|
||||
const estimatedTimeLeftMs = Math.max(0, duration - elapsed);
|
||||
|
||||
return {
|
||||
id: job.job_token,
|
||||
prompt: getPrompt(job, promptsMap),
|
||||
modelId: getModelId(job, promptsMap),
|
||||
modelLabel: getModelLabel(job, promptsMap),
|
||||
progress,
|
||||
estimatedTimeLeftMs,
|
||||
createdAt: job.created_at,
|
||||
promptToken: job.request.maybe_prompt_token ?? undefined,
|
||||
mediaClass,
|
||||
};
|
||||
}
|
||||
|
||||
function jobToFailed(job: Job, promptsMap: Map<string, Prompts>): FailedJob {
|
||||
const failureCategory =
|
||||
job.status.maybe_failure_category_updated ||
|
||||
job.status.maybe_failure_category;
|
||||
const rawMessage =
|
||||
job.status.maybe_failure_message ||
|
||||
job.status.maybe_extra_status_description;
|
||||
const failureReason = failureCategory
|
||||
? FAILURE_REASON_LABEL[failureCategory] || rawMessage || undefined
|
||||
: rawMessage || undefined;
|
||||
const failureMessage =
|
||||
rawMessage && failureCategory !== "unknown" ? rawMessage : undefined;
|
||||
|
||||
const promptToken = job.request.maybe_prompt_token ?? undefined;
|
||||
const promptData = promptToken ? promptsMap.get(promptToken) : undefined;
|
||||
const refImageUrl = pickFirstRefImageUrl(promptData);
|
||||
|
||||
return {
|
||||
id: job.job_token,
|
||||
prompt: getPrompt(job, promptsMap),
|
||||
modelId: getModelId(job, promptsMap),
|
||||
modelLabel: getModelLabel(job, promptsMap),
|
||||
failureReason,
|
||||
failureMessage,
|
||||
status: job.status.status,
|
||||
createdAt: job.created_at,
|
||||
promptToken,
|
||||
refImageUrl,
|
||||
mediaClass: getJobMediaClass(job),
|
||||
};
|
||||
}
|
||||
|
||||
// Semantics that aren't still images — videos and audio clips can't be rendered
|
||||
// as the faded backdrop behind the error state, so skip them when picking the
|
||||
// reference to display.
|
||||
const NON_IMAGE_REF_SEMANTICS = new Set(["vid_ref", "audioref"]);
|
||||
|
||||
function pickFirstRefImageUrl(
|
||||
promptData: Prompts | undefined,
|
||||
): string | undefined {
|
||||
const refs = promptData?.maybe_context_images;
|
||||
if (!refs?.length) return undefined;
|
||||
for (const ref of refs) {
|
||||
if (NON_IMAGE_REF_SEMANTICS.has(ref.semantic)) continue;
|
||||
const url = ref.media_links?.cdn_url;
|
||||
if (url) return url;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function jobToGalleryItem(
|
||||
job: Job,
|
||||
promptsMap?: Map<string, Prompts>,
|
||||
): GalleryItem | null {
|
||||
const result = job.maybe_result;
|
||||
if (!result?.entity_token) return null;
|
||||
|
||||
const mediaClass = getJobMediaClass(job);
|
||||
// Audio has no image thumbnail — its card renders a waveform player.
|
||||
const thumbnail =
|
||||
mediaClass === "audio"
|
||||
? null
|
||||
: getMediaThumbnail(result.media_links, mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
|
||||
return {
|
||||
id: result.entity_token,
|
||||
label: getPrompt(job, promptsMap) || "Generation",
|
||||
thumbnail,
|
||||
fullImage: result.media_links?.cdn_url || null,
|
||||
// Sort by job creation time (not completion time) so the completed card
|
||||
// occupies the same Masonry slot the pending card held — no layout shift.
|
||||
createdAt: job.created_at,
|
||||
mediaClass,
|
||||
modelId: job.request.maybe_model_type ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Expand a single GalleryItem into its batch siblings (if any). */
|
||||
async function expandBatchItems(
|
||||
item: GalleryItem,
|
||||
mediaFilesApi: MediaFilesApi,
|
||||
): Promise<GalleryItem[]> {
|
||||
try {
|
||||
const mediaResponse = await mediaFilesApi.GetMediaFileByToken({
|
||||
mediaFileToken: item.id,
|
||||
});
|
||||
const batchToken = (mediaResponse.data as any)?.maybe_batch_token;
|
||||
if (!batchToken) return [item];
|
||||
|
||||
const batchResponse = await mediaFilesApi.GetMediaFilesByBatchToken({
|
||||
batchToken,
|
||||
});
|
||||
if (!batchResponse.success || !batchResponse.data?.length) return [item];
|
||||
|
||||
return batchResponse.data
|
||||
.map((file: any): GalleryItem | null => {
|
||||
const cdnUrl = file.media_links?.cdn_url;
|
||||
if (!cdnUrl) return null;
|
||||
const thumbnail =
|
||||
item.mediaClass === "audio"
|
||||
? null
|
||||
: getMediaThumbnail(file.media_links, item.mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
}) || cdnUrl;
|
||||
return {
|
||||
id: file.token,
|
||||
label: item.label,
|
||||
thumbnail,
|
||||
fullImage: cdnUrl,
|
||||
createdAt: item.createdAt,
|
||||
mediaClass: item.mediaClass,
|
||||
modelId: item.modelId,
|
||||
batchImageToken: batchToken,
|
||||
durationMillis: file.maybe_duration_millis ?? undefined,
|
||||
};
|
||||
})
|
||||
.filter((i): i is GalleryItem => i !== null);
|
||||
} catch {
|
||||
return [item];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Hook ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export function useGenerationJobs(options: {
|
||||
mediaType: GenerationMediaClass;
|
||||
enabled?: boolean;
|
||||
}) {
|
||||
const { mediaType, enabled = true } = options;
|
||||
const apiRef = useRef(new JobsApi());
|
||||
const mediaApiRef = useRef(new MediaFilesApi());
|
||||
|
||||
const [inProgressJobs, setInProgressJobs] = useState<Job[]>([]);
|
||||
const [failedJobsRaw, setFailedJobsRaw] = useState<Job[]>([]);
|
||||
const [newlyCompleted, setNewlyCompleted] = useState<GalleryItem[]>([]);
|
||||
|
||||
const prevCompletedIdsRef = useRef<Set<string>>(new Set());
|
||||
const initialLoadDoneRef = useRef(false);
|
||||
|
||||
const promptTokens = useMemo(() => {
|
||||
const tokens: string[] = [];
|
||||
for (const j of inProgressJobs) {
|
||||
if (j.request.maybe_prompt_token)
|
||||
tokens.push(j.request.maybe_prompt_token);
|
||||
}
|
||||
for (const j of failedJobsRaw) {
|
||||
if (j.request.maybe_prompt_token)
|
||||
tokens.push(j.request.maybe_prompt_token);
|
||||
}
|
||||
return tokens;
|
||||
}, [inProgressJobs, failedJobsRaw]);
|
||||
const promptsMap = usePrompts(promptTokens);
|
||||
|
||||
const inProgress = useMemo(
|
||||
() =>
|
||||
inProgressJobs
|
||||
.slice()
|
||||
.sort(
|
||||
(a, b) =>
|
||||
new Date(b.created_at).getTime() - new Date(a.created_at).getTime(),
|
||||
)
|
||||
.map((j) => jobToInProgress(j, promptsMap)),
|
||||
[inProgressJobs, promptsMap],
|
||||
);
|
||||
|
||||
const failed = useMemo(
|
||||
() =>
|
||||
failedJobsRaw
|
||||
.slice()
|
||||
.sort(
|
||||
(a, b) =>
|
||||
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
|
||||
)
|
||||
.map((j) => jobToFailed(j, promptsMap)),
|
||||
[failedJobsRaw, promptsMap],
|
||||
);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
const response = await apiRef.current.ListRecentJobs();
|
||||
if (!response.success || !response.data) return;
|
||||
|
||||
const jobs: Job[] = response.data;
|
||||
|
||||
// Filter by media type
|
||||
const filtered = jobs.filter((j) => getJobMediaType(j) === mediaType);
|
||||
|
||||
const newInProgress = filtered.filter((j) =>
|
||||
IN_PROGRESS_STATUSES.has(j.status.status),
|
||||
);
|
||||
const newFailed = filtered.filter((j) =>
|
||||
FAILED_STATUSES.has(j.status.status),
|
||||
);
|
||||
|
||||
// Completed
|
||||
const completedJobs = filtered.filter((j) =>
|
||||
COMPLETED_STATUSES.has(j.status.status),
|
||||
);
|
||||
const completedIdSet = new Set(completedJobs.map((j) => j.job_token));
|
||||
|
||||
// Detect newly completed (skip on first load to avoid flooding)
|
||||
let expandedNewItems: GalleryItem[] = [];
|
||||
if (initialLoadDoneRef.current) {
|
||||
const newOnes = completedJobs.filter(
|
||||
(j) => !prevCompletedIdsRef.current.has(j.job_token),
|
||||
);
|
||||
if (newOnes.length > 0) {
|
||||
const items = newOnes
|
||||
.map((j) => jobToGalleryItem(j))
|
||||
.filter((item): item is GalleryItem => item !== null);
|
||||
if (items.length > 0) {
|
||||
// Await expansion so the pending card and its completed replacement
|
||||
// commit in the same React render — no "remove then add" gap.
|
||||
const expanded = await Promise.all(
|
||||
items.map((item) => expandBatchItems(item, mediaApiRef.current)),
|
||||
);
|
||||
expandedNewItems = expanded.flat();
|
||||
}
|
||||
}
|
||||
}
|
||||
initialLoadDoneRef.current = true;
|
||||
prevCompletedIdsRef.current = completedIdSet;
|
||||
|
||||
// Prune duration cache
|
||||
const activeIds = new Set(newInProgress.map((j) => j.job_token));
|
||||
for (const id of taskDurationCache.keys()) {
|
||||
if (!activeIds.has(id)) taskDurationCache.delete(id);
|
||||
}
|
||||
|
||||
if (expandedNewItems.length > 0) {
|
||||
setNewlyCompleted((prev) => {
|
||||
const existingIds = new Set(prev.map((i) => i.id));
|
||||
const fresh = expandedNewItems.filter((i) => !existingIds.has(i.id));
|
||||
return [...fresh, ...prev];
|
||||
});
|
||||
}
|
||||
setInProgressJobs(newInProgress);
|
||||
setFailedJobsRaw(newFailed);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}, [mediaType]);
|
||||
|
||||
// Poll every 5 seconds + listen for task-queue-update events.
|
||||
// Skip entirely when disabled (e.g. user is logged out) — otherwise we'd
|
||||
// hit an authenticated endpoint every 5s for nothing, which on mobile
|
||||
// Safari shows up as periodic main-thread jank during the menu animation.
|
||||
useEffect(() => {
|
||||
if (!enabled) return;
|
||||
load();
|
||||
const intervalId = setInterval(load, 5000);
|
||||
|
||||
const handleTaskUpdate = () => load();
|
||||
window.addEventListener("task-queue-update", handleTaskUpdate);
|
||||
|
||||
return () => {
|
||||
clearInterval(intervalId);
|
||||
window.removeEventListener("task-queue-update", handleTaskUpdate);
|
||||
};
|
||||
}, [load, enabled]);
|
||||
|
||||
const dismissFailed = useCallback(async (jobToken: string) => {
|
||||
try {
|
||||
await apiRef.current.DeleteJobByToken(jobToken);
|
||||
setFailedJobsRaw((prev) => prev.filter((f) => f.job_token !== jobToken));
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
inProgress,
|
||||
failed,
|
||||
newlyCompleted,
|
||||
dismissFailed,
|
||||
refresh: load,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,9 @@
|
||||
{
|
||||
"path": "../loading-spinner"
|
||||
},
|
||||
{
|
||||
"path": "../audio-player"
|
||||
},
|
||||
{
|
||||
"path": "../../common"
|
||||
},
|
||||
|
||||
@@ -49,6 +49,9 @@
|
||||
{
|
||||
"path": "../loading-spinner/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../audio-player/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../common/tsconfig.lib.json"
|
||||
},
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
faDownToLine,
|
||||
faGlobe,
|
||||
faMagnifyingGlass,
|
||||
faMusic,
|
||||
faPause,
|
||||
faPencil,
|
||||
faPlay,
|
||||
@@ -21,6 +22,7 @@ import {
|
||||
import { MediaFileDelete } from "@storyteller/tauri-api";
|
||||
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
|
||||
import { Viewer3D } from "@storyteller/ui-viewer-3d";
|
||||
import { WaveformAudioPlayer } from "@storyteller/ui-audio-player";
|
||||
import {
|
||||
useEffect,
|
||||
useState,
|
||||
@@ -608,6 +610,19 @@ export function LightboxModal({
|
||||
<source src={selectedImageUrl as string} type="video/mp4" />
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
) : mediaClass === "audio" ? (
|
||||
<div className="flex h-full w-full items-center justify-center px-4 sm:px-8">
|
||||
<div className="w-full max-w-xl rounded-2xl border border-white/10 bg-white/[0.04] px-2 py-8">
|
||||
<FontAwesomeIcon
|
||||
icon={faMusic}
|
||||
className="mx-auto mb-5 block text-3xl text-white/20"
|
||||
/>
|
||||
<WaveformAudioPlayer
|
||||
key={selectedImageUrl as string}
|
||||
src={selectedImageUrl as string}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex h-full w-full flex-col justify-center">
|
||||
<div
|
||||
@@ -692,7 +707,8 @@ export function LightboxModal({
|
||||
|
||||
{!mediaLoaded &&
|
||||
selectedImageUrl &&
|
||||
mediaClass !== "dimensional" && (
|
||||
mediaClass !== "dimensional" &&
|
||||
mediaClass !== "audio" && (
|
||||
<div className="absolute inset-0 bg-ui-panel flex items-center justify-center">
|
||||
<LoadingSpinner className="h-12 w-12 text-base-fg" />
|
||||
</div>
|
||||
|
||||
@@ -29,6 +29,9 @@
|
||||
{
|
||||
"path": "../loading-spinner"
|
||||
},
|
||||
{
|
||||
"path": "../audio-player"
|
||||
},
|
||||
{
|
||||
"path": "../../tauri-api"
|
||||
},
|
||||
|
||||
@@ -70,6 +70,9 @@
|
||||
{
|
||||
"path": "../loading-spinner/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../audio-player/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../tauri-api/tsconfig.lib.json"
|
||||
},
|
||||
|
||||
@@ -3,8 +3,12 @@ export * from "./lib/PromptBox3D";
|
||||
export * from "./lib/PromptBoxEdit";
|
||||
export * from "./lib/PromptBoxVideo";
|
||||
export * from "./lib/PromptBoxImage";
|
||||
export * from "./lib/PromptBoxAudio";
|
||||
export * from "./lib/useAutoGrowEditorHeight";
|
||||
export * from "./lib/PromptFullscreenModal";
|
||||
export * from "./lib/PromptFullscreenButton";
|
||||
export * from "./lib/promptStore";
|
||||
export * from "./lib/PromptBoxErrorBoundary";
|
||||
export * from "./lib/common/AudioTuningPopover";
|
||||
export * from "./lib/common/SoundsSettingsPopover";
|
||||
export * from "./lib/common/StylePromptRow";
|
||||
|
||||
@@ -0,0 +1,498 @@
|
||||
import { useMemo, useRef, useState, ReactNode } from "react";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
import { GenerateButton, ToggleButton } from "@storyteller/ui-button";
|
||||
import {
|
||||
faChevronDown,
|
||||
faChevronUp,
|
||||
faMicrophoneLines,
|
||||
faMicrophoneSlash,
|
||||
faRepeat,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import type { OmniGenAudioModelDetails, UploadMediaFn } from "@storyteller/api";
|
||||
import {
|
||||
enqueueAudioGeneration,
|
||||
AUDIO_MODELS_REQUIRING_AUDIO_REF,
|
||||
OMNI_GENERATE_OUTAGE_MESSAGE,
|
||||
type AudioGenerationSettings,
|
||||
} from "@storyteller/omni-gen";
|
||||
import {
|
||||
getCreatorIconPathForModelId,
|
||||
getModelDescription,
|
||||
getModelInfo,
|
||||
} from "@storyteller/model-list";
|
||||
import { usePromptAudioStore, useEnterToGenerateStore } from "./promptStore";
|
||||
import { useAutoGrowEditorHeight } from "./useAutoGrowEditorHeight";
|
||||
import {
|
||||
PromptFullscreenModal,
|
||||
useFullscreenPrompt,
|
||||
} from "./PromptFullscreenModal";
|
||||
import { PromptFullscreenButton } from "./PromptFullscreenButton";
|
||||
import { gtagEvent } from "@storyteller/google-analytics";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
import { AudioReferenceRow } from "./common/AudioReferenceRow";
|
||||
import { AudioTuningPopover } from "./common/AudioTuningPopover";
|
||||
import { SoundsSettingsPopover } from "./common/SoundsSettingsPopover";
|
||||
import { StylePromptRow } from "./common/StylePromptRow";
|
||||
|
||||
const DEFAULT_AUDIO_MODEL_ID = "suno_music";
|
||||
|
||||
const AUDIO_REF_MAX_DURATION_SECONDS = 600;
|
||||
|
||||
// Capability flags are serde-skipped when absent — only `true` counts.
|
||||
const supports = (flag: boolean | null | undefined): boolean => flag === true;
|
||||
|
||||
interface PromptBoxAudioProps {
|
||||
// Audio models from GET /v1/omni_gen/models/audio (useOmniGenAudioModels).
|
||||
models: OmniGenAudioModelDetails[];
|
||||
uploadAudio?: UploadMediaFn;
|
||||
uploadImage?: UploadMediaFn;
|
||||
// Fired after a successful enqueue with every created job token (one
|
||||
// request can create several Suno clips).
|
||||
onEnqueuePressed?: (jobTokens: string[]) => void | Promise<void>;
|
||||
credits?: number | null;
|
||||
}
|
||||
|
||||
export const PromptBoxAudio = ({
|
||||
models,
|
||||
uploadAudio,
|
||||
uploadImage,
|
||||
onEnqueuePressed,
|
||||
credits,
|
||||
}: PromptBoxAudioProps) => {
|
||||
const prompt = usePromptAudioStore((s) => s.prompt);
|
||||
const setPrompt = usePromptAudioStore((s) => s.setPrompt);
|
||||
const stylePrompt = usePromptAudioStore((s) => s.stylePrompt);
|
||||
const setStylePrompt = usePromptAudioStore((s) => s.setStylePrompt);
|
||||
const selectedModelId = usePromptAudioStore((s) => s.selectedModelId);
|
||||
const setSelectedModelId = usePromptAudioStore((s) => s.setSelectedModelId);
|
||||
const isInstrumental = usePromptAudioStore((s) => s.isInstrumental);
|
||||
const setIsInstrumental = usePromptAudioStore((s) => s.setIsInstrumental);
|
||||
const keepLyrics = usePromptAudioStore((s) => s.keepLyrics);
|
||||
const setKeepLyrics = usePromptAudioStore((s) => s.setKeepLyrics);
|
||||
const isLoopable = usePromptAudioStore((s) => s.isLoopable);
|
||||
const setIsLoopable = usePromptAudioStore((s) => s.setIsLoopable);
|
||||
const bpm = usePromptAudioStore((s) => s.bpm);
|
||||
const setBpm = usePromptAudioStore((s) => s.setBpm);
|
||||
const musicalKey = usePromptAudioStore((s) => s.musicalKey);
|
||||
const setMusicalKey = usePromptAudioStore((s) => s.setMusicalKey);
|
||||
const sampleRateHz = usePromptAudioStore((s) => s.sampleRateHz);
|
||||
const setSampleRateHz = usePromptAudioStore((s) => s.setSampleRateHz);
|
||||
const speed = usePromptAudioStore((s) => s.speed);
|
||||
const setSpeed = usePromptAudioStore((s) => s.setSpeed);
|
||||
const volume = usePromptAudioStore((s) => s.volume);
|
||||
const setVolume = usePromptAudioStore((s) => s.setVolume);
|
||||
const pitch = usePromptAudioStore((s) => s.pitch);
|
||||
const setPitch = usePromptAudioStore((s) => s.setPitch);
|
||||
const referenceAudios = usePromptAudioStore((s) => s.referenceAudios);
|
||||
const setReferenceAudios = usePromptAudioStore((s) => s.setReferenceAudios);
|
||||
const referenceImages = usePromptAudioStore((s) => s.referenceImages);
|
||||
const setReferenceImages = usePromptAudioStore((s) => s.setReferenceImages);
|
||||
|
||||
const [isEnqueueing, setIsEnqueueing] = useState(false);
|
||||
const [isFocused, setIsFocused] = useState(false);
|
||||
const enterToGenerate = useEnterToGenerateStore((s) => s.enabled);
|
||||
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const { isExpanded, toggleExpand } = useAutoGrowEditorHeight(
|
||||
textareaRef,
|
||||
prompt,
|
||||
);
|
||||
const { isFullscreen, openFullscreen, closeFullscreen } =
|
||||
useFullscreenPrompt();
|
||||
|
||||
const selectedModel = useMemo((): OmniGenAudioModelDetails | undefined => {
|
||||
if (!models.length) return undefined;
|
||||
if (selectedModelId) {
|
||||
const chosen = models.find((m) => m.model === selectedModelId);
|
||||
if (chosen) return chosen;
|
||||
}
|
||||
return (
|
||||
models.find((m) => m.model === DEFAULT_AUDIO_MODEL_ID) ?? models[0]
|
||||
);
|
||||
}, [models, selectedModelId]);
|
||||
|
||||
// Capability gating
|
||||
const styleSupported = supports(selectedModel?.style_prompt_supported);
|
||||
const hasInstrumental = supports(
|
||||
selectedModel?.instrumental_toggle_supported,
|
||||
);
|
||||
const hasKeepLyrics = supports(selectedModel?.keep_lyrics_supported);
|
||||
const hasLoopable = supports(selectedModel?.loopable_toggle_supported);
|
||||
const hasBpm = supports(selectedModel?.bpm_supported);
|
||||
const hasMusicalKey = supports(selectedModel?.musical_key_supported);
|
||||
const sampleRateOptions = selectedModel?.sample_rate_hz_options ?? null;
|
||||
const hasSpeed = supports(selectedModel?.speed_supported);
|
||||
const hasVolume = supports(selectedModel?.volume_supported);
|
||||
const hasPitch = supports(selectedModel?.pitch_supported);
|
||||
const hasTuning =
|
||||
!!sampleRateOptions?.length || hasSpeed || hasVolume || hasPitch;
|
||||
const audioRefsSupported = supports(
|
||||
selectedModel?.audio_references_supported,
|
||||
);
|
||||
const maxAudioRefs = selectedModel?.audio_references_max ?? 1;
|
||||
const imageRefsSupported = supports(
|
||||
selectedModel?.image_references_supported,
|
||||
);
|
||||
const requiresAudioRef = AUDIO_MODELS_REQUIRING_AUDIO_REF.has(
|
||||
selectedModel?.model ?? "",
|
||||
);
|
||||
const missingRequiredAudioRef =
|
||||
requiresAudioRef && referenceAudios.length !== 1;
|
||||
|
||||
const effectiveSampleRate =
|
||||
sampleRateHz != null && sampleRateOptions?.includes(sampleRateHz)
|
||||
? sampleRateHz
|
||||
: (selectedModel?.sample_rate_hz_default ??
|
||||
sampleRateOptions?.[0] ??
|
||||
null);
|
||||
|
||||
const modelItems = useMemo(
|
||||
(): PopoverItem[] =>
|
||||
models.map((model) => ({
|
||||
label: model.full_name || model.model,
|
||||
selected: model.model === selectedModel?.model,
|
||||
description: getModelDescription(model.model, model.extra_info_short),
|
||||
info: getModelInfo(model.model, model.extra_info) || undefined,
|
||||
icon: (
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(model.model)}
|
||||
alt={`${model.model} logo`}
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
),
|
||||
action: model.model,
|
||||
})),
|
||||
[models, selectedModel?.model],
|
||||
);
|
||||
|
||||
// Seed Audio can't combine audio and image references — adding one kind
|
||||
// clears the other so the request is always valid.
|
||||
const handleReferenceAudiosChange = (audios: typeof referenceAudios) => {
|
||||
if (audios.length > 0 && referenceImages.length > 0) {
|
||||
setReferenceImages([]);
|
||||
toast.error("Removed image reference — it can't be combined with audio");
|
||||
}
|
||||
setReferenceAudios(audios);
|
||||
};
|
||||
|
||||
const handleReferenceImagesChange = (images: typeof referenceImages) => {
|
||||
if (images.length > 0 && referenceAudios.length > 0) {
|
||||
setReferenceAudios([]);
|
||||
toast.error("Removed audio reference — it can't be combined with an image");
|
||||
}
|
||||
setReferenceImages(images);
|
||||
};
|
||||
|
||||
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
||||
setPrompt(e.target.value);
|
||||
};
|
||||
|
||||
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
|
||||
e.preventDefault();
|
||||
const pastedText = e.clipboardData.getData("text");
|
||||
const target = e.currentTarget;
|
||||
const { selectionStart, selectionEnd, value } = target;
|
||||
const next =
|
||||
value.slice(0, selectionStart) + pastedText + value.slice(selectionEnd);
|
||||
setPrompt(next);
|
||||
requestAnimationFrame(() => {
|
||||
const pos = Math.min(selectionStart + pastedText.length, next.length);
|
||||
textareaRef.current?.setSelectionRange(pos, pos);
|
||||
});
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
if (e.key !== "Enter") return;
|
||||
if (enterToGenerate && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
handleEnqueue();
|
||||
}
|
||||
};
|
||||
|
||||
const handleEnqueue = async () => {
|
||||
if (!prompt.trim() || !selectedModel || isEnqueueing) return;
|
||||
|
||||
if (missingRequiredAudioRef) {
|
||||
toast.error(
|
||||
`${selectedModel.full_name ?? "This model"} needs an audio track to work from — add one first`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
setIsEnqueueing(true);
|
||||
gtagEvent("enqueue_audio");
|
||||
|
||||
try {
|
||||
const settings: AudioGenerationSettings = {
|
||||
prompt,
|
||||
stylePrompt,
|
||||
audioMediaTokens: referenceAudios
|
||||
.map((audio) => audio.mediaToken)
|
||||
.filter((token) => token.length > 0),
|
||||
imageMediaTokens: referenceImages
|
||||
.map((image) => image.mediaToken)
|
||||
.filter((token) => token.length > 0),
|
||||
isInstrumental,
|
||||
keepLyrics,
|
||||
isLoopable,
|
||||
bpm,
|
||||
musicalKey,
|
||||
sampleRateHz: effectiveSampleRate,
|
||||
speed,
|
||||
volume,
|
||||
pitch,
|
||||
};
|
||||
|
||||
const result = await enqueueAudioGeneration(selectedModel, settings);
|
||||
|
||||
if (!result.success) {
|
||||
if (result.errorCode === 402) {
|
||||
toast.error("Not enough credits for this generation");
|
||||
} else if (result.errorCode != null && result.errorCode >= 500) {
|
||||
toast.error(OMNI_GENERATE_OUTAGE_MESSAGE);
|
||||
} else {
|
||||
toast.error(result.error ?? "Failed to start audio generation");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await onEnqueuePressed?.(result.jobTokens);
|
||||
} catch (err) {
|
||||
console.error("PromptBoxAudio - enqueue failed", err);
|
||||
toast.error("Failed to start audio generation. Please try again.");
|
||||
} finally {
|
||||
setIsEnqueueing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const modelSelector: ReactNode = (
|
||||
<Tooltip content="Model" position="top" className="z-50" closeOnClick>
|
||||
<PopoverMenu
|
||||
items={modelItems}
|
||||
onSelect={(item) => {
|
||||
if (item.action) setSelectedModelId(item.action);
|
||||
}}
|
||||
mode="toggle"
|
||||
panelTitle="Select Model"
|
||||
panelClassName="w-[360px]"
|
||||
richList
|
||||
triggerIcon={
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
|
||||
alt=""
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
}
|
||||
triggerLabel={selectedModel?.full_name ?? "Model"}
|
||||
/>
|
||||
</Tooltip>
|
||||
);
|
||||
|
||||
const toggleButtons: ReactNode = (
|
||||
<>
|
||||
{hasInstrumental && (
|
||||
<Tooltip
|
||||
content={isInstrumental ? "Instrumental: ON" : "Instrumental: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={isInstrumental}
|
||||
icon={faMicrophoneSlash}
|
||||
activeIcon={faMicrophoneSlash}
|
||||
label="Instrumental"
|
||||
onClick={() => setIsInstrumental(!isInstrumental)}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
{hasKeepLyrics && (
|
||||
<Tooltip
|
||||
content={keepLyrics ? "Keep lyrics: ON" : "Keep lyrics: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={keepLyrics}
|
||||
icon={faMicrophoneLines}
|
||||
activeIcon={faMicrophoneLines}
|
||||
label="Keep lyrics"
|
||||
onClick={() => setKeepLyrics(!keepLyrics)}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
{hasLoopable && (
|
||||
<Tooltip
|
||||
content={isLoopable ? "Loop: ON" : "Loop: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={isLoopable}
|
||||
icon={faRepeat}
|
||||
activeIcon={faRepeat}
|
||||
label="Loop"
|
||||
onClick={() => setIsLoopable(!isLoopable)}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const settingsPopovers: ReactNode = (
|
||||
<>
|
||||
{(hasBpm || hasMusicalKey) && (
|
||||
<SoundsSettingsPopover
|
||||
showBpm={hasBpm}
|
||||
bpm={bpm}
|
||||
onBpmChange={setBpm}
|
||||
showMusicalKey={hasMusicalKey}
|
||||
musicalKey={musicalKey}
|
||||
onMusicalKeyChange={setMusicalKey}
|
||||
/>
|
||||
)}
|
||||
{hasTuning && (
|
||||
<AudioTuningPopover
|
||||
sampleRateOptions={sampleRateOptions}
|
||||
sampleRateHz={effectiveSampleRate}
|
||||
onSampleRateChange={setSampleRateHz}
|
||||
showSpeed={hasSpeed}
|
||||
speed={speed}
|
||||
onSpeedChange={setSpeed}
|
||||
showVolume={hasVolume}
|
||||
volume={volume}
|
||||
onVolumeChange={setVolume}
|
||||
showPitch={hasPitch}
|
||||
pitch={pitch}
|
||||
onPitchChange={setPitch}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const referenceRow: ReactNode =
|
||||
audioRefsSupported || imageRefsSupported ? (
|
||||
<AudioReferenceRow
|
||||
referenceAudios={referenceAudios}
|
||||
onReferenceAudiosChange={handleReferenceAudiosChange}
|
||||
maxAudioCount={audioRefsSupported ? maxAudioRefs : 0}
|
||||
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
|
||||
uploadAudio={uploadAudio}
|
||||
audioRequired={requiresAudioRef}
|
||||
imageSupported={imageRefsSupported}
|
||||
referenceImages={referenceImages}
|
||||
onReferenceImagesChange={handleReferenceImagesChange}
|
||||
uploadImage={uploadImage}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="relative z-20 flex flex-col">
|
||||
<div
|
||||
className={twMerge(
|
||||
"glass relative w-full rounded-2xl p-4",
|
||||
isFocused
|
||||
? "ring-1 ring-primary border-primary"
|
||||
: "ring-1 ring-transparent",
|
||||
)}
|
||||
>
|
||||
{referenceRow}
|
||||
|
||||
<div className="flex justify-center gap-2">
|
||||
<div className="promptbox-resize-wrap relative flex-1">
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
rows={1}
|
||||
placeholder="Describe the music or sound you want..."
|
||||
className="promptbox-scrollbar text-md mb-2 min-h-[2.5em] w-full resize-y overflow-y-auto rounded bg-transparent pb-2 pr-8 pt-1 text-base-fg placeholder-base-fg/60 transition-[height] duration-200 ease-out focus:outline-none"
|
||||
value={prompt}
|
||||
onChange={handleChange}
|
||||
onPaste={handlePaste}
|
||||
onKeyDown={handleKeyDown}
|
||||
onFocus={() => setIsFocused(true)}
|
||||
onBlur={() => setIsFocused(false)}
|
||||
/>
|
||||
<PromptFullscreenButton onClick={openFullscreen} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{styleSupported && (
|
||||
<StylePromptRow value={stylePrompt} onChange={setStylePrompt} />
|
||||
)}
|
||||
|
||||
<div className="mt-2 flex items-center justify-between gap-2">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{modelSelector}
|
||||
{toggleButtons}
|
||||
{settingsPopovers}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{missingRequiredAudioRef && (
|
||||
<span className="flex animate-pulse items-center gap-1.5 text-xs font-medium text-red-500">
|
||||
Audio track required
|
||||
</span>
|
||||
)}
|
||||
<GenerateButton
|
||||
className="flex items-center border-none bg-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
|
||||
icon={undefined}
|
||||
onClick={handleEnqueue}
|
||||
disabled={!prompt.trim() || missingRequiredAudioRef}
|
||||
loading={isEnqueueing}
|
||||
credits={credits}
|
||||
>
|
||||
Generate
|
||||
</GenerateButton>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="absolute -bottom-1 left-1/2 -translate-x-1/2">
|
||||
<Tooltip
|
||||
content={isExpanded ? "Collapse" : "Expand"}
|
||||
position="top"
|
||||
className="-mb-2"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggleExpand}
|
||||
className="text-base-fg/30 hover:text-base-fg/90 transition-colors px-3 py-0.5"
|
||||
>
|
||||
<FontAwesomeIcon
|
||||
icon={isExpanded ? faChevronUp : faChevronDown}
|
||||
className="text-xs"
|
||||
/>
|
||||
</button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<PromptFullscreenModal
|
||||
isOpen={isFullscreen}
|
||||
onClose={closeFullscreen}
|
||||
promptLength={prompt.length}
|
||||
maxLength={Infinity}
|
||||
footerControls={
|
||||
<>
|
||||
{modelSelector}
|
||||
{toggleButtons}
|
||||
{settingsPopovers}
|
||||
</>
|
||||
}
|
||||
>
|
||||
<textarea
|
||||
placeholder="Describe the music or sound you want..."
|
||||
className="promptbox-scrollbar text-md h-full min-h-0 w-full resize-none overflow-y-auto rounded bg-transparent text-base-fg placeholder-base-fg/60 focus:outline-none"
|
||||
value={prompt}
|
||||
onChange={handleChange}
|
||||
onPaste={handlePaste}
|
||||
onKeyDown={handleKeyDown}
|
||||
/>
|
||||
</PromptFullscreenModal>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,329 @@
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faImage,
|
||||
faMusic,
|
||||
faPlay,
|
||||
faPlus,
|
||||
faStop,
|
||||
faXmark,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
import { UploaderStates } from "@storyteller/common";
|
||||
import type { UploadMediaFn } from "@storyteller/api";
|
||||
import type { RefAudio, RefImage } from "../promptStore";
|
||||
|
||||
export interface AudioReferenceRowProps {
|
||||
referenceAudios: RefAudio[];
|
||||
onReferenceAudiosChange: (audios: RefAudio[]) => void;
|
||||
maxAudioCount: number;
|
||||
maxAudioRefDuration: number;
|
||||
uploadAudio?: UploadMediaFn;
|
||||
// Whether the audio reference is required (remix/sample source).
|
||||
audioRequired?: boolean;
|
||||
// Optional single-image reference section (Seed Audio).
|
||||
imageSupported?: boolean;
|
||||
referenceImages?: RefImage[];
|
||||
onReferenceImagesChange?: (images: RefImage[]) => void;
|
||||
uploadImage?: UploadMediaFn;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
// Reference row for the audio promptbox: upload an audio track (the
|
||||
// remix/sample source or Seed Audio refs), plus an optional image reference
|
||||
// for models that support one. Rendered inside the glass card, above the
|
||||
// prompt textarea.
|
||||
export function AudioReferenceRow({
|
||||
referenceAudios,
|
||||
onReferenceAudiosChange,
|
||||
maxAudioCount,
|
||||
maxAudioRefDuration,
|
||||
uploadAudio,
|
||||
audioRequired = false,
|
||||
imageSupported = false,
|
||||
referenceImages = [],
|
||||
onReferenceImagesChange,
|
||||
uploadImage,
|
||||
className = "",
|
||||
}: AudioReferenceRowProps) {
|
||||
const audioInputRef = useRef<HTMLInputElement>(null);
|
||||
const imageInputRef = useRef<HTMLInputElement>(null);
|
||||
const [isUploadingAudio, setIsUploadingAudio] = useState(false);
|
||||
const [isUploadingImage, setIsUploadingImage] = useState(false);
|
||||
|
||||
// Latest-value refs so async upload callbacks append to fresh state.
|
||||
const referenceAudiosRef = useRef(referenceAudios);
|
||||
referenceAudiosRef.current = referenceAudios;
|
||||
const referenceImagesRef = useRef(referenceImages);
|
||||
referenceImagesRef.current = referenceImages;
|
||||
|
||||
const handleAudioFileUpload = async (
|
||||
event: React.ChangeEvent<HTMLInputElement>,
|
||||
) => {
|
||||
const files = Array.from(event.target.files || []);
|
||||
if (files.length === 0) return;
|
||||
|
||||
const availableSlots = Math.max(0, maxAudioCount - referenceAudios.length);
|
||||
const filesToProcess = files.slice(0, availableSlots);
|
||||
|
||||
for (const file of filesToProcess) {
|
||||
const duration = await getAudioFileDuration(file);
|
||||
const currentTotal = referenceAudiosRef.current.reduce(
|
||||
(sum, audio) => sum + audio.duration,
|
||||
0,
|
||||
);
|
||||
if (currentTotal + duration > maxAudioRefDuration) {
|
||||
toast.error(
|
||||
`Total audio duration cannot exceed ${maxAudioRefDuration}s`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
if (uploadAudio) {
|
||||
setIsUploadingAudio(true);
|
||||
await uploadAudio({
|
||||
title: `reference-audio-${Math.random().toString(36).substring(2, 15)}`,
|
||||
assetFile: file,
|
||||
progressCallback: (newState) => {
|
||||
if (newState.status === UploaderStates.success && newState.data) {
|
||||
const refAudio: RefAudio = {
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: URL.createObjectURL(file),
|
||||
file,
|
||||
mediaToken: newState.data,
|
||||
duration,
|
||||
};
|
||||
setIsUploadingAudio(false);
|
||||
onReferenceAudiosChange([
|
||||
...referenceAudiosRef.current,
|
||||
refAudio,
|
||||
]);
|
||||
} else if (
|
||||
newState.status === UploaderStates.assetError ||
|
||||
newState.status === UploaderStates.imageCreateError
|
||||
) {
|
||||
setIsUploadingAudio(false);
|
||||
toast.error("Audio upload failed. Please try again.");
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (audioInputRef.current) audioInputRef.current.value = "";
|
||||
};
|
||||
|
||||
const handleImageFileUpload = async (
|
||||
event: React.ChangeEvent<HTMLInputElement>,
|
||||
) => {
|
||||
const file = (event.target.files || [])[0];
|
||||
if (!file || !uploadImage) return;
|
||||
|
||||
setIsUploadingImage(true);
|
||||
await uploadImage({
|
||||
title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
|
||||
assetFile: file,
|
||||
progressCallback: (newState) => {
|
||||
if (newState.status === UploaderStates.success && newState.data) {
|
||||
const refImage: RefImage = {
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: URL.createObjectURL(file),
|
||||
file,
|
||||
mediaToken: newState.data,
|
||||
};
|
||||
setIsUploadingImage(false);
|
||||
onReferenceImagesChange?.([refImage]);
|
||||
} else if (
|
||||
newState.status === UploaderStates.assetError ||
|
||||
newState.status === UploaderStates.imageCreateError
|
||||
) {
|
||||
setIsUploadingImage(false);
|
||||
toast.error("Image upload failed. Please try again.");
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (imageInputRef.current) imageInputRef.current.value = "";
|
||||
};
|
||||
|
||||
const removeAudio = (id: string) => {
|
||||
onReferenceAudiosChange(referenceAudios.filter((a) => a.id !== id));
|
||||
if (audioInputRef.current) audioInputRef.current.value = "";
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={`flex flex-col gap-2 pb-2.5 ${className}`}>
|
||||
<input
|
||||
ref={audioInputRef}
|
||||
type="file"
|
||||
accept="audio/*"
|
||||
className="hidden"
|
||||
multiple={maxAudioCount > 1}
|
||||
onChange={handleAudioFileUpload}
|
||||
/>
|
||||
{imageSupported && (
|
||||
<input
|
||||
ref={imageInputRef}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
className="hidden"
|
||||
onChange={handleImageFileUpload}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<div className="flex items-center gap-2 text-base-fg opacity-90">
|
||||
<FontAwesomeIcon icon={faMusic} className="h-3.5 w-3.5" />
|
||||
<span className="text-sm font-medium">
|
||||
Audio Track{" "}
|
||||
<span className="font-semibold text-base-fg/60">
|
||||
({referenceAudios.length}/{maxAudioCount})
|
||||
</span>
|
||||
{audioRequired && referenceAudios.length === 0 && (
|
||||
<span className="ml-1.5 text-xs font-medium text-red-500">
|
||||
required
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{referenceAudios.map((audio, index) => (
|
||||
<AudioRefTile
|
||||
key={audio.id}
|
||||
audio={audio}
|
||||
index={index}
|
||||
onRemove={removeAudio}
|
||||
/>
|
||||
))}
|
||||
|
||||
{referenceAudios.length < maxAudioCount && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => audioInputRef.current?.click()}
|
||||
disabled={isUploadingAudio}
|
||||
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
|
||||
>
|
||||
<FontAwesomeIcon icon={faPlus} className="h-3 w-3" />
|
||||
{isUploadingAudio ? "Uploading…" : "Add audio"}
|
||||
</button>
|
||||
)}
|
||||
|
||||
{imageSupported && (
|
||||
<>
|
||||
<div className="ms-2 flex items-center gap-2 text-base-fg opacity-90">
|
||||
<FontAwesomeIcon icon={faImage} className="h-3.5 w-3.5" />
|
||||
<span className="text-sm font-medium">Image</span>
|
||||
</div>
|
||||
{referenceImages.map((image) => (
|
||||
<div
|
||||
key={image.id}
|
||||
className="group relative h-9 w-9 overflow-hidden rounded-lg border border-ui-controls-border"
|
||||
>
|
||||
<img
|
||||
src={image.url}
|
||||
alt="Reference"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Remove image"
|
||||
onClick={() => onReferenceImagesChange?.([])}
|
||||
className="absolute inset-0 flex items-center justify-center bg-black/60 opacity-0 transition-opacity group-hover:opacity-100"
|
||||
>
|
||||
<FontAwesomeIcon
|
||||
icon={faXmark}
|
||||
className="h-3 w-3 text-white"
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
{referenceImages.length === 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => imageInputRef.current?.click()}
|
||||
disabled={isUploadingImage}
|
||||
className="flex h-9 items-center gap-1.5 rounded-lg border border-dashed border-ui-controls-border bg-ui-controls/50 px-3 text-sm text-base-fg/70 transition-colors hover:bg-ui-controls hover:text-base-fg disabled:cursor-wait disabled:opacity-60"
|
||||
>
|
||||
<FontAwesomeIcon icon={faPlus} className="h-3 w-3" />
|
||||
{isUploadingImage ? "Uploading…" : "Add image"}
|
||||
</button>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function AudioRefTile({
|
||||
audio,
|
||||
index,
|
||||
onRemove,
|
||||
}: {
|
||||
audio: RefAudio;
|
||||
index: number;
|
||||
onRemove: (id: string) => void;
|
||||
}) {
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
|
||||
const togglePlay = useCallback(() => {
|
||||
if (isPlaying) {
|
||||
audioRef.current?.pause();
|
||||
audioRef.current = null;
|
||||
setIsPlaying(false);
|
||||
return;
|
||||
}
|
||||
const element = new Audio(audio.url);
|
||||
element.onended = () => {
|
||||
audioRef.current = null;
|
||||
setIsPlaying(false);
|
||||
};
|
||||
audioRef.current = element;
|
||||
void element.play().catch(() => setIsPlaying(false));
|
||||
setIsPlaying(true);
|
||||
}, [audio.url, isPlaying]);
|
||||
|
||||
return (
|
||||
<div className="group flex h-9 items-center gap-2 rounded-lg border border-ui-controls-border bg-ui-controls px-2.5">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={isPlaying ? "Stop" : "Play"}
|
||||
onClick={togglePlay}
|
||||
className="flex h-5 w-5 items-center justify-center rounded-full bg-white/90 text-black transition-transform hover:scale-105"
|
||||
>
|
||||
<FontAwesomeIcon
|
||||
icon={isPlaying ? faStop : faPlay}
|
||||
className={`h-2 w-2 ${isPlaying ? "" : "ml-px"}`}
|
||||
/>
|
||||
</button>
|
||||
<span className="text-xs font-medium text-base-fg/80">
|
||||
Audio {index + 1} · {audio.duration}s
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Remove audio"
|
||||
onClick={() => onRemove(audio.id)}
|
||||
className="flex h-4 w-4 items-center justify-center rounded text-base-fg/40 transition-colors hover:text-base-fg"
|
||||
>
|
||||
<FontAwesomeIcon icon={faXmark} className="h-2.5 w-2.5" />
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getAudioFileDuration(file: File): Promise<number> {
|
||||
return new Promise((resolve) => {
|
||||
const audio = document.createElement("audio");
|
||||
audio.preload = "metadata";
|
||||
audio.onloadedmetadata = () => {
|
||||
URL.revokeObjectURL(audio.src);
|
||||
resolve(Math.round(audio.duration));
|
||||
};
|
||||
audio.onerror = () => {
|
||||
URL.revokeObjectURL(audio.src);
|
||||
resolve(0);
|
||||
};
|
||||
audio.src = URL.createObjectURL(file);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faSliders } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { PopoverMenu } from "@storyteller/ui-popover";
|
||||
import { SliderV2 } from "@storyteller/ui-sliderv2";
|
||||
|
||||
// Output-shaping controls for models that support them (Seed Audio):
|
||||
// sample rate, playback speed, volume, and pitch. Ranges mirror the
|
||||
// backend's validate_audio_request limits.
|
||||
export const AUDIO_SPEED_MIN = 0.5;
|
||||
export const AUDIO_SPEED_MAX = 2.0;
|
||||
export const AUDIO_VOLUME_MIN = 0.5;
|
||||
export const AUDIO_VOLUME_MAX = 2.0;
|
||||
export const AUDIO_PITCH_MIN = -12;
|
||||
export const AUDIO_PITCH_MAX = 12;
|
||||
|
||||
export interface AudioTuningPopoverProps {
|
||||
// Sample rate section renders only when options are provided.
|
||||
sampleRateOptions?: number[] | null;
|
||||
sampleRateHz?: number | null;
|
||||
onSampleRateChange?: (hz: number) => void;
|
||||
showSpeed?: boolean;
|
||||
speed?: number;
|
||||
onSpeedChange?: (value: number) => void;
|
||||
showVolume?: boolean;
|
||||
volume?: number;
|
||||
onVolumeChange?: (value: number) => void;
|
||||
showPitch?: boolean;
|
||||
pitch?: number;
|
||||
onPitchChange?: (value: number) => void;
|
||||
}
|
||||
|
||||
export function AudioTuningPopover({
|
||||
sampleRateOptions,
|
||||
sampleRateHz,
|
||||
onSampleRateChange,
|
||||
showSpeed = false,
|
||||
speed = 1,
|
||||
onSpeedChange,
|
||||
showVolume = false,
|
||||
volume = 1,
|
||||
onVolumeChange,
|
||||
showPitch = false,
|
||||
pitch = 0,
|
||||
onPitchChange,
|
||||
}: AudioTuningPopoverProps) {
|
||||
const showSampleRate =
|
||||
!!sampleRateOptions?.length && onSampleRateChange !== undefined;
|
||||
|
||||
return (
|
||||
<PopoverMenu
|
||||
mode="default"
|
||||
panelTitle="Tuning"
|
||||
triggerIcon={<FontAwesomeIcon icon={faSliders} className="h-3.5 w-3.5" />}
|
||||
triggerLabel="Tuning"
|
||||
>
|
||||
<div className="w-60 space-y-3.5 pb-0.5">
|
||||
{showSampleRate && (
|
||||
<div>
|
||||
<div className="mb-1.5 text-xs font-medium text-base-fg/60">
|
||||
Sample rate
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{sampleRateOptions.map((hz) => (
|
||||
<button
|
||||
key={hz}
|
||||
type="button"
|
||||
onClick={() => onSampleRateChange?.(hz)}
|
||||
className={`rounded-md border px-2 py-1 text-xs font-medium transition-colors ${
|
||||
hz === sampleRateHz
|
||||
? "border-white/30 bg-white/15 text-base-fg"
|
||||
: "border-transparent bg-white/5 text-base-fg/60 hover:bg-white/10 hover:text-base-fg"
|
||||
}`}
|
||||
>
|
||||
{formatSampleRateHz(hz)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showSpeed && (
|
||||
<TuningSliderRow
|
||||
label="Speed"
|
||||
valueLabel={`${speed.toFixed(2)}×`}
|
||||
min={AUDIO_SPEED_MIN}
|
||||
max={AUDIO_SPEED_MAX}
|
||||
step={0.05}
|
||||
value={speed}
|
||||
onChange={(value) => onSpeedChange?.(value)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showVolume && (
|
||||
<TuningSliderRow
|
||||
label="Volume"
|
||||
valueLabel={`${volume.toFixed(2)}×`}
|
||||
min={AUDIO_VOLUME_MIN}
|
||||
max={AUDIO_VOLUME_MAX}
|
||||
step={0.05}
|
||||
value={volume}
|
||||
onChange={(value) => onVolumeChange?.(value)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showPitch && (
|
||||
<TuningSliderRow
|
||||
label="Pitch"
|
||||
valueLabel={`${pitch > 0 ? "+" : ""}${pitch} st`}
|
||||
min={AUDIO_PITCH_MIN}
|
||||
max={AUDIO_PITCH_MAX}
|
||||
step={1}
|
||||
value={pitch}
|
||||
onChange={(value) => onPitchChange?.(value)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</PopoverMenu>
|
||||
);
|
||||
}
|
||||
|
||||
interface TuningSliderRowProps {
|
||||
label: string;
|
||||
valueLabel: string;
|
||||
min: number;
|
||||
max: number;
|
||||
step: number;
|
||||
value: number;
|
||||
onChange: (value: number) => void;
|
||||
}
|
||||
|
||||
function TuningSliderRow({
|
||||
label,
|
||||
valueLabel,
|
||||
min,
|
||||
max,
|
||||
step,
|
||||
value,
|
||||
onChange,
|
||||
}: TuningSliderRowProps) {
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-base-fg/60">{label}</span>
|
||||
<span className="text-xs font-medium tabular-nums text-base-fg">
|
||||
{valueLabel}
|
||||
</span>
|
||||
</div>
|
||||
<SliderV2
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
variant="filled"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function formatSampleRateHz(hz: number): string {
|
||||
const kilohertz = hz / 1000;
|
||||
const label = Number.isInteger(kilohertz)
|
||||
? kilohertz.toString()
|
||||
: kilohertz.toFixed(1);
|
||||
return `${label} kHz`;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faDrum } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { PopoverMenu } from "@storyteller/ui-popover";
|
||||
import { SliderV2 } from "@storyteller/ui-sliderv2";
|
||||
|
||||
// UI-only bounds; the server accepts any u16 BPM.
|
||||
export const AUDIO_BPM_MIN = 40;
|
||||
export const AUDIO_BPM_MAX = 240;
|
||||
const DEFAULT_BPM = 120;
|
||||
|
||||
// Musical keys for audio generation (eg. Suno Sounds).
|
||||
// NB: There are intentionally no E keys, per product spec.
|
||||
export const MUSICAL_KEYS: Array<{
|
||||
value: string;
|
||||
label: string;
|
||||
shortLabel: string;
|
||||
}> = [
|
||||
{ value: "auto", label: "Auto", shortLabel: "Auto" },
|
||||
{ value: "c_major", label: "C Major", shortLabel: "C" },
|
||||
{ value: "c_minor", label: "C Minor", shortLabel: "Cm" },
|
||||
{ value: "d_major", label: "D Major", shortLabel: "D" },
|
||||
{ value: "d_minor", label: "D Minor", shortLabel: "Dm" },
|
||||
{ value: "f_major", label: "F Major", shortLabel: "F" },
|
||||
{ value: "f_minor", label: "F Minor", shortLabel: "Fm" },
|
||||
{ value: "g_major", label: "G Major", shortLabel: "G" },
|
||||
{ value: "g_minor", label: "G Minor", shortLabel: "Gm" },
|
||||
{ value: "a_major", label: "A Major", shortLabel: "A" },
|
||||
{ value: "a_minor", label: "A Minor", shortLabel: "Am" },
|
||||
{ value: "b_major", label: "B Major", shortLabel: "B" },
|
||||
{ value: "b_minor", label: "B Minor", shortLabel: "Bm" },
|
||||
];
|
||||
|
||||
export interface SoundsSettingsPopoverProps {
|
||||
showBpm?: boolean;
|
||||
// Null means "let the model decide" — the field is omitted from the request.
|
||||
bpm?: number | null;
|
||||
onBpmChange?: (bpm: number | null) => void;
|
||||
showMusicalKey?: boolean;
|
||||
musicalKey?: string;
|
||||
onMusicalKeyChange?: (key: string) => void;
|
||||
}
|
||||
|
||||
export function SoundsSettingsPopover({
|
||||
showBpm = false,
|
||||
bpm = null,
|
||||
onBpmChange,
|
||||
showMusicalKey = false,
|
||||
musicalKey = "auto",
|
||||
onMusicalKeyChange,
|
||||
}: SoundsSettingsPopoverProps) {
|
||||
const keyLabel =
|
||||
MUSICAL_KEYS.find((entry) => entry.value === musicalKey)?.shortLabel ??
|
||||
"Auto";
|
||||
const triggerLabel = [
|
||||
showBpm ? (bpm === null ? "Auto BPM" : `${bpm} BPM`) : null,
|
||||
showMusicalKey ? keyLabel : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
|
||||
return (
|
||||
<PopoverMenu
|
||||
mode="default"
|
||||
panelTitle="Beat & Key"
|
||||
triggerIcon={<FontAwesomeIcon icon={faDrum} className="h-3.5 w-3.5" />}
|
||||
triggerLabel={triggerLabel || "Beat"}
|
||||
>
|
||||
<div className="w-60 space-y-3.5 pb-0.5">
|
||||
{showBpm && (
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-base-fg/60">BPM</span>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="text-xs font-medium tabular-nums text-base-fg">
|
||||
{bpm === null ? "Auto" : bpm}
|
||||
</span>
|
||||
{bpm !== null && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onBpmChange?.(null)}
|
||||
className="rounded bg-white/5 px-1.5 py-0.5 text-[10px] font-medium text-base-fg/60 transition-colors hover:bg-white/10 hover:text-base-fg"
|
||||
>
|
||||
Auto
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<SliderV2
|
||||
min={AUDIO_BPM_MIN}
|
||||
max={AUDIO_BPM_MAX}
|
||||
step={1}
|
||||
value={bpm ?? DEFAULT_BPM}
|
||||
onChange={(value) => onBpmChange?.(value)}
|
||||
variant="filled"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showMusicalKey && (
|
||||
<div>
|
||||
<div className="mb-1.5 text-xs font-medium text-base-fg/60">
|
||||
Musical key
|
||||
</div>
|
||||
<div className="grid grid-cols-4 gap-1">
|
||||
{MUSICAL_KEYS.map((entry) => (
|
||||
<button
|
||||
key={entry.value}
|
||||
type="button"
|
||||
title={entry.label}
|
||||
onClick={() => onMusicalKeyChange?.(entry.value)}
|
||||
className={`rounded-md border px-1.5 py-1 text-xs font-medium transition-colors ${
|
||||
entry.value === musicalKey
|
||||
? "border-white/30 bg-white/15 text-base-fg"
|
||||
: "border-transparent bg-white/5 text-base-fg/60 hover:bg-white/10 hover:text-base-fg"
|
||||
} ${entry.value === "auto" ? "col-span-4" : ""}`}
|
||||
>
|
||||
{entry.shortLabel}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</PopoverMenu>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { ChangeEvent } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faTags, faXmark } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
|
||||
export interface StylePromptRowProps {
|
||||
value: string;
|
||||
onChange: (value: string) => void;
|
||||
placeholder?: string;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
// Always-visible secondary prompt row for the style/genre direction
|
||||
// (Suno's "tags"), rendered directly beneath the main prompt textarea.
|
||||
export function StylePromptRow({
|
||||
value,
|
||||
onChange,
|
||||
placeholder = "Style: e.g. dreamy synth-pop, female vocals",
|
||||
className = "",
|
||||
}: StylePromptRowProps) {
|
||||
const handleChange = (event: ChangeEvent<HTMLInputElement>) => {
|
||||
onChange(event.target.value);
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
className={twMerge(
|
||||
"flex items-center gap-2 border-t border-white/10 px-1 pt-2",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<FontAwesomeIcon
|
||||
icon={faTags}
|
||||
className="h-3.5 w-3.5 shrink-0 text-base-fg/40"
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
value={value}
|
||||
onChange={handleChange}
|
||||
placeholder={placeholder}
|
||||
className="min-w-0 flex-1 bg-transparent text-sm text-base-fg placeholder:text-base-fg/35 focus:outline-none"
|
||||
/>
|
||||
{value.length > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Clear style"
|
||||
onClick={() => onChange("")}
|
||||
className="flex h-5 w-5 shrink-0 items-center justify-center rounded text-base-fg/40 transition-colors hover:bg-white/10 hover:text-base-fg"
|
||||
>
|
||||
<FontAwesomeIcon icon={faXmark} className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -186,6 +186,75 @@ export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
|
||||
setGenerationCount: (generationCount) => set({ generationCount }),
|
||||
}));
|
||||
|
||||
// ----- Audio Prompt Box Store -----
|
||||
interface PromptAudioStore {
|
||||
prompt: string;
|
||||
// Style/genre direction (Suno's "tags").
|
||||
stylePrompt: string;
|
||||
// Model id ("suno_music", ...) — audio models come from the omni HTTP API,
|
||||
// not the ClassyModelSelector store.
|
||||
selectedModelId: string | null;
|
||||
// Suno toggles
|
||||
isInstrumental: boolean;
|
||||
keepLyrics: boolean;
|
||||
isLoopable: boolean;
|
||||
// Suno Sounds beat controls. bpm null = "Auto" (omitted from the request).
|
||||
bpm: number | null;
|
||||
musicalKey: string;
|
||||
// Seed Audio output shaping.
|
||||
sampleRateHz: number | null;
|
||||
speed: number;
|
||||
volume: number;
|
||||
pitch: number;
|
||||
referenceAudios: RefAudio[];
|
||||
referenceImages: RefImage[];
|
||||
setPrompt: (prompt: string) => void;
|
||||
setStylePrompt: (stylePrompt: string) => void;
|
||||
setSelectedModelId: (modelId: string | null) => void;
|
||||
setIsInstrumental: (value: boolean) => void;
|
||||
setKeepLyrics: (value: boolean) => void;
|
||||
setIsLoopable: (value: boolean) => void;
|
||||
setBpm: (bpm: number | null) => void;
|
||||
setMusicalKey: (musicalKey: string) => void;
|
||||
setSampleRateHz: (sampleRateHz: number | null) => void;
|
||||
setSpeed: (speed: number) => void;
|
||||
setVolume: (volume: number) => void;
|
||||
setPitch: (pitch: number) => void;
|
||||
setReferenceAudios: (audios: RefAudio[]) => void;
|
||||
setReferenceImages: (images: RefImage[]) => void;
|
||||
}
|
||||
|
||||
export const usePromptAudioStore = create<PromptAudioStore>()((set) => ({
|
||||
prompt: "",
|
||||
stylePrompt: "",
|
||||
selectedModelId: null,
|
||||
isInstrumental: false,
|
||||
keepLyrics: false,
|
||||
isLoopable: false,
|
||||
bpm: null,
|
||||
musicalKey: "auto",
|
||||
sampleRateHz: null,
|
||||
speed: 1,
|
||||
volume: 1,
|
||||
pitch: 0,
|
||||
referenceAudios: [],
|
||||
referenceImages: [],
|
||||
setPrompt: (prompt) => set({ prompt }),
|
||||
setStylePrompt: (stylePrompt) => set({ stylePrompt }),
|
||||
setSelectedModelId: (selectedModelId) => set({ selectedModelId }),
|
||||
setIsInstrumental: (isInstrumental) => set({ isInstrumental }),
|
||||
setKeepLyrics: (keepLyrics) => set({ keepLyrics }),
|
||||
setIsLoopable: (isLoopable) => set({ isLoopable }),
|
||||
setBpm: (bpm) => set({ bpm }),
|
||||
setMusicalKey: (musicalKey) => set({ musicalKey }),
|
||||
setSampleRateHz: (sampleRateHz) => set({ sampleRateHz }),
|
||||
setSpeed: (speed) => set({ speed }),
|
||||
setVolume: (volume) => set({ volume }),
|
||||
setPitch: (pitch) => set({ pitch }),
|
||||
setReferenceAudios: (referenceAudios) => set({ referenceAudios }),
|
||||
setReferenceImages: (referenceImages) => set({ referenceImages }),
|
||||
}));
|
||||
|
||||
// ----- Edit Prompt Box Store -----
|
||||
type EditAspectRatio = "auto" | "wide" | "tall" | "square";
|
||||
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
{
|
||||
"path": "../button-icon-select"
|
||||
},
|
||||
{
|
||||
"path": "../../omni-gen"
|
||||
},
|
||||
{
|
||||
"path": "../../analytics/google-analytics"
|
||||
},
|
||||
|
||||
@@ -49,6 +49,9 @@
|
||||
{
|
||||
"path": "../button-icon-select/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../omni-gen/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../../analytics/google-analytics/tsconfig.lib.json"
|
||||
},
|
||||
|
||||
@@ -22,4 +22,5 @@ export enum ModelCreator {
|
||||
Grok = "Grok",
|
||||
WorldLabs = "WorldLabs",
|
||||
Beeble = "Beeble",
|
||||
Suno = "Suno",
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ export const MODEL_ID_PREFIX_CREATORS: Array<[string, ModelCreator]> = [
|
||||
["grok", ModelCreator.Grok],
|
||||
["happy_horse", ModelCreator.Alibaba],
|
||||
["qwen", ModelCreator.Alibaba],
|
||||
["suno", ModelCreator.Suno],
|
||||
["seed_audio", ModelCreator.Bytedance],
|
||||
// Beeble SwitchX (background change) has no provider icon — use the ArtCraft
|
||||
// mark since it's surfaced as an ArtCraft feature.
|
||||
["switch_x", ModelCreator.ArtCraft],
|
||||
|
||||
@@ -26,6 +26,7 @@ const CREATOR_ICON_FILES: Partial<Record<ModelCreator, string>> = {
|
||||
[ModelCreator.ArtCraft]: "artcraft.svg",
|
||||
[ModelCreator.Grok]: "grok.svg",
|
||||
[ModelCreator.WorldLabs]: "worldlabs.svg",
|
||||
[ModelCreator.Suno]: "suno.svg",
|
||||
};
|
||||
|
||||
export const getCreatorIconPath = (creator: ModelCreator): string => {
|
||||
|
||||
@@ -41,6 +41,12 @@ const MODEL_DESCRIPTIONS: Record<string, string> = {
|
||||
veo_3p1_fast: "Speed-tuned Veo 3.1",
|
||||
// ── Edit / VFX ──
|
||||
switch_x: "Swap or relight backgrounds",
|
||||
// ── Audio models ──
|
||||
suno_music: "Full songs from a text prompt",
|
||||
suno_remix: "Remix an existing track",
|
||||
suno_sounds: "Sound effects with beat control",
|
||||
suno_sample: "Build a song from a sample",
|
||||
seed_audio_1p0: "Sound generation with fine tuning",
|
||||
};
|
||||
|
||||
const MODEL_INFOS: Record<string, string> = {
|
||||
|
||||
@@ -100,6 +100,15 @@ export const getModelDisplayName = (
|
||||
seedream_4p5: "Seedream 4.5",
|
||||
seedream_5_lite: "Seedream 5 Lite",
|
||||
|
||||
// Audio — Suno
|
||||
suno_music: "Suno Music",
|
||||
suno_remix: "Suno Remix",
|
||||
suno_sounds: "Suno Sounds",
|
||||
suno_sample: "Suno Sample",
|
||||
|
||||
// Audio — Seed Audio (ByteDance)
|
||||
seed_audio_1p0: "Seed Audio 1.0",
|
||||
|
||||
// Google
|
||||
veo_2: "Google Veo 2",
|
||||
veo_3: "Google Veo 3",
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
export * from "./lib/omni-gen-hooks.js";
|
||||
export * from "./lib/omni-gen-audio.js";
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OmniGenAudioModelDetails } from "@storyteller/api";
|
||||
import { buildAudioRequest, type AudioGenerationSettings } from "./omni-gen-audio";
|
||||
|
||||
// Per-model capability fixtures mirroring the backend's audio_models config.
|
||||
const SUNO_MUSIC: OmniGenAudioModelDetails = {
|
||||
model: "suno_music",
|
||||
text_prompt_supported: true,
|
||||
style_prompt_supported: true,
|
||||
instrumental_toggle_supported: true,
|
||||
};
|
||||
|
||||
const SUNO_REMIX: OmniGenAudioModelDetails = {
|
||||
model: "suno_remix",
|
||||
text_prompt_supported: true,
|
||||
style_prompt_supported: true,
|
||||
keep_lyrics_supported: true,
|
||||
audio_references_supported: true,
|
||||
audio_references_max: 1,
|
||||
};
|
||||
|
||||
const SUNO_SOUNDS: OmniGenAudioModelDetails = {
|
||||
model: "suno_sounds",
|
||||
text_prompt_supported: true,
|
||||
loopable_toggle_supported: true,
|
||||
bpm_supported: true,
|
||||
musical_key_supported: true,
|
||||
};
|
||||
|
||||
const SEED_AUDIO: OmniGenAudioModelDetails = {
|
||||
model: "seed_audio_1p0",
|
||||
text_prompt_supported: true,
|
||||
audio_references_supported: true,
|
||||
audio_references_max: 3,
|
||||
image_references_supported: true,
|
||||
image_references_max: 1,
|
||||
sample_rate_hz_options: [8000, 16000, 24000, 32000, 44100, 48000],
|
||||
sample_rate_hz_default: 24000,
|
||||
speed_supported: true,
|
||||
volume_supported: true,
|
||||
pitch_supported: true,
|
||||
};
|
||||
|
||||
const BASE_SETTINGS: AudioGenerationSettings = {
|
||||
prompt: "a driving synthwave anthem",
|
||||
stylePrompt: "dreamy synth-pop, female vocals",
|
||||
audioMediaTokens: ["mf_audio1"],
|
||||
imageMediaTokens: ["mf_image1"],
|
||||
isInstrumental: true,
|
||||
keepLyrics: true,
|
||||
isLoopable: true,
|
||||
bpm: 128,
|
||||
musicalKey: "a_minor",
|
||||
sampleRateHz: 44100,
|
||||
speed: 1.25,
|
||||
volume: 0.8,
|
||||
pitch: -2,
|
||||
};
|
||||
|
||||
describe("buildAudioRequest capability gating", () => {
|
||||
it("suno_music sends prompts + instrumental but no refs or beat/tuning params", () => {
|
||||
const request = buildAudioRequest(SUNO_MUSIC, BASE_SETTINGS);
|
||||
expect(request.model).toBe("suno_music");
|
||||
expect(request.prompt).toBe(BASE_SETTINGS.prompt);
|
||||
expect(request.style_prompt).toBe(BASE_SETTINGS.stylePrompt);
|
||||
expect(request.is_instrumental).toBe(true);
|
||||
expect(request.audio_media_tokens).toBeNull();
|
||||
expect(request.image_media_tokens).toBeNull();
|
||||
expect(request.keep_lyrics).toBeNull();
|
||||
expect(request.is_loopable).toBeNull();
|
||||
expect(request.bpm).toBeNull();
|
||||
expect(request.musical_key).toBeNull();
|
||||
expect(request.sample_rate_hz).toBeNull();
|
||||
expect(request.speed).toBeNull();
|
||||
expect(request.volume).toBeNull();
|
||||
expect(request.pitch).toBeNull();
|
||||
});
|
||||
|
||||
it("suno_remix sends the audio reference + keep_lyrics but never style-less extras", () => {
|
||||
const request = buildAudioRequest(SUNO_REMIX, BASE_SETTINGS);
|
||||
expect(request.audio_media_tokens).toEqual(["mf_audio1"]);
|
||||
expect(request.keep_lyrics).toBe(true);
|
||||
expect(request.image_media_tokens).toBeNull();
|
||||
expect(request.is_instrumental).toBeNull();
|
||||
expect(request.bpm).toBeNull();
|
||||
});
|
||||
|
||||
it("suno_sounds sends beat controls but never a style prompt", () => {
|
||||
const request = buildAudioRequest(SUNO_SOUNDS, BASE_SETTINGS);
|
||||
expect(request.style_prompt).toBeNull();
|
||||
expect(request.is_loopable).toBe(true);
|
||||
expect(request.bpm).toBe(128);
|
||||
expect(request.musical_key).toBe("a_minor");
|
||||
expect(request.audio_media_tokens).toBeNull();
|
||||
});
|
||||
|
||||
it("suno_sounds omits bpm when set to Auto (null)", () => {
|
||||
const request = buildAudioRequest(SUNO_SOUNDS, {
|
||||
...BASE_SETTINGS,
|
||||
bpm: null,
|
||||
});
|
||||
expect(request.bpm).toBeNull();
|
||||
expect(request.musical_key).toBe("a_minor");
|
||||
});
|
||||
|
||||
it("seed_audio sends tuning params + refs but never Suno toggles", () => {
|
||||
const request = buildAudioRequest(SEED_AUDIO, BASE_SETTINGS);
|
||||
expect(request.sample_rate_hz).toBe(44100);
|
||||
expect(request.speed).toBe(1.25);
|
||||
expect(request.volume).toBe(0.8);
|
||||
expect(request.pitch).toBe(-2);
|
||||
expect(request.audio_media_tokens).toEqual(["mf_audio1"]);
|
||||
expect(request.image_media_tokens).toEqual(["mf_image1"]);
|
||||
expect(request.is_instrumental).toBeNull();
|
||||
expect(request.keep_lyrics).toBeNull();
|
||||
expect(request.is_loopable).toBeNull();
|
||||
expect(request.style_prompt).toBeNull();
|
||||
});
|
||||
|
||||
it("omits empty prompts and reference arrays", () => {
|
||||
const request = buildAudioRequest(SUNO_MUSIC, {
|
||||
...BASE_SETTINGS,
|
||||
prompt: " ",
|
||||
stylePrompt: "",
|
||||
audioMediaTokens: [],
|
||||
imageMediaTokens: [],
|
||||
});
|
||||
expect(request.prompt).toBeNull();
|
||||
expect(request.style_prompt).toBeNull();
|
||||
expect(request.audio_media_tokens).toBeNull();
|
||||
});
|
||||
|
||||
it("always includes a fresh idempotency token", () => {
|
||||
const first = buildAudioRequest(SUNO_MUSIC, BASE_SETTINGS);
|
||||
const second = buildAudioRequest(SUNO_MUSIC, BASE_SETTINGS);
|
||||
expect(first.idempotency_token).toBeTruthy();
|
||||
expect(second.idempotency_token).toBeTruthy();
|
||||
expect(first.idempotency_token).not.toBe(second.idempotency_token);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { OmniGenApi } from "@storyteller/api";
|
||||
import type {
|
||||
OmniGenAudioModelDetails,
|
||||
OmniGenAudioRequest,
|
||||
OmniGenMusicalKey,
|
||||
} from "@storyteller/api";
|
||||
|
||||
// Audio request building + enqueue + cost estimation, shared by the webapp
|
||||
// create-audio page and the desktop PromptBoxAudio (both enqueue over HTTP).
|
||||
|
||||
// Server rule (validate_audio_request): these models require exactly one
|
||||
// audio reference (the remix/sample source). The models API has no
|
||||
// "required" flag, so mirror it by id.
|
||||
export const AUDIO_MODELS_REQUIRING_AUDIO_REF = new Set([
|
||||
"suno_remix",
|
||||
"suno_sample",
|
||||
]);
|
||||
|
||||
// ── Request params ───────────────────────────────────────────────────────
|
||||
|
||||
// Raw UI state. buildAudioRequest gates every field on the selected model's
|
||||
// capability flags so unsupported settings never reach the API.
|
||||
export interface AudioGenerationSettings {
|
||||
prompt: string;
|
||||
stylePrompt: string;
|
||||
audioMediaTokens: string[];
|
||||
imageMediaTokens: string[];
|
||||
isInstrumental: boolean;
|
||||
keepLyrics: boolean;
|
||||
isLoopable: boolean;
|
||||
bpm: number | null;
|
||||
musicalKey: string;
|
||||
sampleRateHz: number | null;
|
||||
speed: number;
|
||||
volume: number;
|
||||
pitch: number;
|
||||
}
|
||||
|
||||
// ── Request builder ──────────────────────────────────────────────────────
|
||||
|
||||
// Capability flags are serde-skipped when absent, so only `=== true` counts
|
||||
// as supported.
|
||||
const supports = (flag: boolean | null | undefined): boolean => flag === true;
|
||||
|
||||
export function buildAudioRequest(
|
||||
model: OmniGenAudioModelDetails,
|
||||
settings: AudioGenerationSettings,
|
||||
): OmniGenAudioRequest {
|
||||
return {
|
||||
model: model.model,
|
||||
idempotency_token: crypto.randomUUID(),
|
||||
prompt: settings.prompt.trim() || null,
|
||||
style_prompt:
|
||||
supports(model.style_prompt_supported) && settings.stylePrompt.trim()
|
||||
? settings.stylePrompt.trim()
|
||||
: null,
|
||||
audio_media_tokens:
|
||||
supports(model.audio_references_supported) &&
|
||||
settings.audioMediaTokens.length
|
||||
? settings.audioMediaTokens
|
||||
: null,
|
||||
image_media_tokens:
|
||||
supports(model.image_references_supported) &&
|
||||
settings.imageMediaTokens.length
|
||||
? settings.imageMediaTokens
|
||||
: null,
|
||||
keep_lyrics: supports(model.keep_lyrics_supported)
|
||||
? settings.keepLyrics
|
||||
: null,
|
||||
is_instrumental: supports(model.instrumental_toggle_supported)
|
||||
? settings.isInstrumental
|
||||
: null,
|
||||
is_loopable: supports(model.loopable_toggle_supported)
|
||||
? settings.isLoopable
|
||||
: null,
|
||||
bpm:
|
||||
supports(model.bpm_supported) && settings.bpm !== null
|
||||
? settings.bpm
|
||||
: null,
|
||||
musical_key: supports(model.musical_key_supported)
|
||||
? (settings.musicalKey as OmniGenMusicalKey)
|
||||
: null,
|
||||
sample_rate_hz: model.sample_rate_hz_options?.length
|
||||
? settings.sampleRateHz
|
||||
: null,
|
||||
speed: supports(model.speed_supported) ? settings.speed : null,
|
||||
volume: supports(model.volume_supported) ? settings.volume : null,
|
||||
pitch: supports(model.pitch_supported) ? settings.pitch : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Enqueue generation ───────────────────────────────────────────────────
|
||||
|
||||
export async function enqueueAudioGeneration(
|
||||
model: OmniGenAudioModelDetails,
|
||||
settings: AudioGenerationSettings,
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
// One request can create several jobs (Suno-style multi-clip) — callers
|
||||
// must treat every token as its own pending generation.
|
||||
jobTokens: string[];
|
||||
error?: string;
|
||||
errorCode?: number;
|
||||
}> {
|
||||
const body = buildAudioRequest(model, settings);
|
||||
|
||||
try {
|
||||
const api = new OmniGenApi();
|
||||
const response = await api.generateAudio(body);
|
||||
if (response.success && response.inference_job_token) {
|
||||
const jobTokens = response.all_job_tokens?.length
|
||||
? response.all_job_tokens
|
||||
: [response.inference_job_token];
|
||||
return { success: true, jobTokens };
|
||||
}
|
||||
return { success: false, jobTokens: [], error: "Generation failed" };
|
||||
} catch (err: any) {
|
||||
return {
|
||||
success: false,
|
||||
jobTokens: [],
|
||||
error: err.message ?? "Request failed",
|
||||
errorCode: parseAudioHttpStatusCode(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Pull the HTTP status out of an ApiManager error. ApiManager throws
|
||||
// `Error("HTTP error! status: 402")` on a non-2xx response (the JSON body is
|
||||
// not surfaced), so callers can special-case codes like 402 Payment Required.
|
||||
export function parseAudioHttpStatusCode(err: unknown): number | undefined {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const match = /status:\s*(\d+)/.exec(message);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
// ── Cost estimate hook ───────────────────────────────────────────────────
|
||||
|
||||
export interface AudioCostParams {
|
||||
model: string;
|
||||
audioReferenceCount: number;
|
||||
hasImageReference: boolean;
|
||||
sampleRateHz?: number | null;
|
||||
}
|
||||
|
||||
export function useAudioCostEstimate(params: AudioCostParams): number | null {
|
||||
const [credits, setCredits] = useState<number | null>(null);
|
||||
const abortRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!params.model) {
|
||||
setCredits(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const id = ++abortRef.current;
|
||||
|
||||
const body: OmniGenAudioRequest = {
|
||||
model: params.model,
|
||||
audio_media_tokens: params.audioReferenceCount
|
||||
? new Array(params.audioReferenceCount).fill("placeholder")
|
||||
: null,
|
||||
image_media_tokens: params.hasImageReference ? ["placeholder"] : null,
|
||||
sample_rate_hz: params.sampleRateHz ?? null,
|
||||
};
|
||||
|
||||
const api = new OmniGenApi();
|
||||
api.estimateAudioCost(body).then(
|
||||
(response) => {
|
||||
if (id !== abortRef.current) return;
|
||||
if (response.success && response.cost_in_credits != null) {
|
||||
setCredits(response.cost_in_credits);
|
||||
} else {
|
||||
setCredits(null);
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (id !== abortRef.current) return;
|
||||
setCredits(null);
|
||||
},
|
||||
);
|
||||
}, [
|
||||
params.model,
|
||||
params.audioReferenceCount,
|
||||
params.hasImageReference,
|
||||
params.sampleRateHz,
|
||||
]);
|
||||
|
||||
return credits;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { OmniGenApi } from "@storyteller/api";
|
||||
import type {
|
||||
OmniGenAudioModelDetails,
|
||||
OmniGenImageModelInfo,
|
||||
OmniGenVideoModelInfo,
|
||||
} from "@storyteller/api";
|
||||
@@ -38,6 +39,11 @@ let videoModelsFetching = false;
|
||||
let videoModelsError: string | null = null;
|
||||
let videoModelsListeners: Array<() => void> = [];
|
||||
|
||||
let audioModelsCache: OmniGenAudioModelDetails[] | null = null;
|
||||
let audioModelsFetching = false;
|
||||
let audioModelsError: string | null = null;
|
||||
let audioModelsListeners: Array<() => void> = [];
|
||||
|
||||
function notifyImageListeners() {
|
||||
imageModelsListeners.forEach((cb) => cb());
|
||||
imageModelsListeners = [];
|
||||
@@ -48,6 +54,11 @@ function notifyVideoListeners() {
|
||||
videoModelsListeners = [];
|
||||
}
|
||||
|
||||
function notifyAudioListeners() {
|
||||
audioModelsListeners.forEach((cb) => cb());
|
||||
audioModelsListeners = [];
|
||||
}
|
||||
|
||||
function fetchImageModelsOnce() {
|
||||
if (imageModelsCache || imageModelsFetching) return;
|
||||
imageModelsFetching = true;
|
||||
@@ -102,6 +113,33 @@ function fetchVideoModelsOnce() {
|
||||
);
|
||||
}
|
||||
|
||||
function fetchAudioModelsOnce() {
|
||||
if (audioModelsCache || audioModelsFetching) return;
|
||||
audioModelsFetching = true;
|
||||
audioModelsError = null;
|
||||
const api = new OmniGenApi();
|
||||
api.getAudioModels().then(
|
||||
(res) => {
|
||||
audioModelsFetching = false;
|
||||
if (res.success && res.models?.length) {
|
||||
audioModelsCache = res.models.filter((m) => m.is_disabled !== true);
|
||||
} else {
|
||||
audioModelsError = "No audio models returned from API";
|
||||
console.warn("[OmniGen] Audio models response:", res);
|
||||
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
|
||||
}
|
||||
notifyAudioListeners();
|
||||
},
|
||||
(err) => {
|
||||
audioModelsFetching = false;
|
||||
audioModelsError = err?.message ?? "Failed to fetch audio models";
|
||||
console.error("[OmniGen] Failed to fetch audio models:", err);
|
||||
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
|
||||
notifyAudioListeners();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// ── Hooks ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function useOmniGenImageModels(): {
|
||||
@@ -187,3 +225,45 @@ export function useOmniGenVideoModels(): {
|
||||
|
||||
return { models, isLoading, error };
|
||||
}
|
||||
|
||||
export function useOmniGenAudioModels(): {
|
||||
models: OmniGenAudioModelDetails[];
|
||||
isLoading: boolean;
|
||||
error: string | null;
|
||||
} {
|
||||
const [models, setModels] = useState<OmniGenAudioModelDetails[]>(
|
||||
audioModelsCache ?? [],
|
||||
);
|
||||
const [isLoading, setIsLoading] = useState(!audioModelsCache);
|
||||
const [error, setError] = useState<string | null>(audioModelsError);
|
||||
|
||||
useEffect(() => {
|
||||
if (audioModelsCache) {
|
||||
setModels(audioModelsCache);
|
||||
setIsLoading(false);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const onReady = () => {
|
||||
setIsLoading(false);
|
||||
if (audioModelsCache) {
|
||||
setModels(audioModelsCache);
|
||||
setError(null);
|
||||
} else {
|
||||
setError(audioModelsError);
|
||||
}
|
||||
};
|
||||
|
||||
audioModelsListeners.push(onReady);
|
||||
fetchAudioModelsOnce();
|
||||
|
||||
return () => {
|
||||
audioModelsListeners = audioModelsListeners.filter(
|
||||
(cb) => cb !== onReady,
|
||||
);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { models, isLoading, error };
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ export function isExternal(id: string): boolean {
|
||||
id === 'konva' ||
|
||||
id === 'three' ||
|
||||
id === 'zustand' ||
|
||||
id === 'wavesurfer.js' ||
|
||||
id.startsWith('@fortawesome/')
|
||||
) {
|
||||
return true;
|
||||
|
||||
Generated
+179
-89
File diff suppressed because it is too large
Load Diff
@@ -215,6 +215,9 @@
|
||||
},
|
||||
{
|
||||
"path": "./libs/omni-gen"
|
||||
},
|
||||
{
|
||||
"path": "./libs/components/audio-player"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user