diff --git a/Cargo.lock b/Cargo.lock index 3e06681256..c1992afe18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,28 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.6.1" @@ -196,9 +218,9 @@ dependencies = [ "jni 0.22.4", "libc", "log", - "ndk", + "ndk 0.9.0", "ndk-context", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "num_enum", "thiserror 2.0.18", ] @@ -480,6 +502,7 @@ dependencies = [ "reqwest 0.12.23", "resvg", "rfd", + "rodio", "serde", "serde_json", "tokens", @@ -1730,6 +1753,49 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk 0.8.0", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -1930,6 +1996,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.5.0" @@ -2696,6 +2768,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fal_client" version = "0.0.1" @@ -4832,6 +4910,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "markup5ever" version = "0.35.0" @@ -5075,6 +5162,20 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys 0.5.0+25.2.9519653", + "num_enum", + "thiserror 1.0.63", +] + [[package]] name = "ndk" version = "0.9.0" @@ -5084,7 +5185,7 @@ dependencies = [ "bitflags 2.13.2", "jni-sys 0.3.1", "log", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "num_enum", "raw-window-handle", "thiserror 1.0.63", @@ -5096,6 +5197,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -5642,6 +5752,29 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni 0.21.1", + "ndk 0.8.0", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -6917,6 +7050,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rodio" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ceb6607dd738c99bc8cb28eff249b7cd5c8ec88b9db96c0608c1480d140fb1" +dependencies = [ + "cpal", + "symphonia", +] + [[package]] name = "ron" version = "0.12.2" @@ -8258,6 +8401,151 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "1.0.109" @@ -8366,8 +8654,8 @@ dependencies = [ "jni 0.21.1", "libc", "log", - "ndk", - "ndk-sys", + "ndk 0.9.0", + "ndk-sys 0.6.0+11769913", "objc2 0.6.4", "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", @@ -10447,6 +10735,16 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.61.3" @@ -10490,6 +10788,16 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.61.2" @@ -10603,6 +10911,15 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -10985,7 +11302,7 @@ dependencies = [ "js-sys", "libc", "memmap2", - "ndk", + "ndk 0.9.0", "objc2 0.5.2", "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", @@ -11296,7 +11613,7 @@ dependencies = [ "javascriptcore-rs", "jni 0.21.1", "libc", - "ndk", + "ndk 0.9.0", "objc2 0.6.4", "objc2-app-kit 0.3.2", "objc2-core-foundation", diff --git a/artcraft-native/Cargo.toml b/artcraft-native/Cargo.toml index d2def584f7..603f2941dd 100644 --- a/artcraft-native/Cargo.toml +++ b/artcraft-native/Cargo.toml @@ -31,6 +31,7 @@ open = "5" reqwest = { workspace = true, features = ["multipart"] } resvg = "0.45" rfd = "0.16" +rodio = { version = "0.20", default-features = false, features = ["symphonia-all"] } serde.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] } diff --git a/artcraft-native/README.md b/artcraft-native/README.md index 73b2fcc322..87b266e37b 100644 --- a/artcraft-native/README.md +++ b/artcraft-native/README.md @@ -7,8 +7,8 @@ The look follows the brutalist webapp (`artcraft-services/frontend/apps/artcraft means near-black surfaces with white/15 hairlines and 3 px corners on controls. Type is Archivo for display, Inter for body text and Geist Mono for labels. The brand blue marks the primary actions. -Create Image and Create Video are fully built. The other tools show a "coming soon" page that -links to the same tool in the web app. +Create Image, Create Video and Create Audio are fully built. The other tools show a "coming soon" +page that links to the same tool in the web app. ## Running @@ -32,9 +32,10 @@ The workspace's default member is the Tauri app, so pass `-p artcraft_native`. | `backend::wire` | Lenient response shapes; new server enum values can't break decoding | | `backend::session` | Credentials: own file, Tauri cookie import, env config | | `backend::media_cache` | Thumbnails and previews: downloaded and decoded off-thread, evicted when unused | +| `backend::audio` | One clip at a time: reference previews, feed tracks (pause, seek, volume), waveforms | | `models` | Model catalog from OmniGen listings, plus names, families and limits | | `prompt_box` | The shared prompt box: editor, reference deck, keyframes, pickers, model selector | -| `pages` | Create Image, Create Video, Home, Library, and the shared page shell | +| `pages` | Create Image, Video and Audio, Home, Library, and the shared page shell | | `feed` | Generation feed: running, failed and finished jobs, library paging, grid and list | | `overlays` | Lightbox, library picker, sign-in and settings dialogs | | `shell` | Custom title bar, sidebar flush to the window edge, content panel header | @@ -42,12 +43,13 @@ The workspace's default member is the Tauri app, so pass `-p artcraft_native`. How the backend works: -- **Generation.** It goes through OmniGen: `/v1/omni_gen/{models,cost,generate}/{image,video}`. +- **Generation.** It goes through OmniGen: `/v1/omni_gen/{models,cost,generate}/{image,video,audio}`. Model capabilities come from the server, so new models and options appear without an app update. - **Typed client functions.** Model lists, uploads, credits, login and deletes use `artcraft_client`. - **Lenient decoding.** Generate and cost requests are JSON maps, as in the Tauri app. Job polling - (`/v1/jobs/session`), the library list, batches and prompts are decoded with lenient structs. + (`/v1/jobs/session`), the audio model list, the library list, batches and prompts are decoded + with lenient structs. - **Identity headers.** Every request sends the desktop identity headers: `Origin` and `User-Agent`. @@ -69,23 +71,29 @@ How the backend works: | Live credit cost on the generate button | Done | | Validation: starting frame, text-only banner, limits, uploads | Done | | Clear all (confirms when references are attached) | Done | -| `@Character` mentions and the Characters modal (Seedance 2.0) | Not yet | +| `@Character` mentions and the Characters modal (create/edit/delete) | Done | | Provider choice (Midjourney direct, fal) | Not yet; everything runs via ArtCraft | -| In-app video and audio playback | Opens in the system player instead | -| Animated video previews in the feed | Still frames only | +| In-app video playback | Done with FFmpeg on the PATH; otherwise opens in the system player | +| Audio reference playback (click to play/stop) | Done | +| Animated video previews in the feed (with a stills toggle) | Done | +| Audio: style, instrumental, keep lyrics, loop | Done | +| Audio: Beat & Key (BPM, musical key) and Tuning (sample rate, speed, volume, pitch) | Done | +| Audio references (play, 600 s cap) or one image, never both | Done | Feed and lightbox: - **Feed.** Pending cards show progress, time left and a batch banner. Failed cards show the reason and can be dismissed. The grid is masonry, with a list view alongside. Hover actions are Recreate, Make Video, Share and Download. Select mode does batch downloads. The last viewed item is marked. +- **Audio.** Audio cards play in place: a waveform (click to seek), the time and a volume flyout. + One track plays at a time and stops when you leave the page. Audio offers Share and Download. - **Lightbox.** Prev/next with the arrow keys, prompt copy and details. Actions are Recreate, Make - Video, Download, Share and Delete. + Video, Download, Share and Delete. Audio plays in a panel with the full-size player. ## Diagnostics Set `ARTCRAFT_SCREENSHOT=` to save a screenshot once the window has settled. -`ARTCRAFT_PAGE` (`home`, `create-image`, `create-video` or `library`) picks the page, and +`ARTCRAFT_PAGE` (`home`, `create-image`, `create-video`, `create-audio` or `library`) picks the page, and `ARTCRAFT_SCREENSHOT_EXIT=1` closes the app afterwards. ## Tests @@ -95,8 +103,8 @@ cargo test -p artcraft_native ``` There are unit tests for the catalog, feed store, wire decoding and backend helpers. Headless UI -tests (`egui_kittest`) drive the prompt box: generate gating, Enter to generate, the pickers, -`@` mentions and clear-all. +tests (`egui_kittest`) drive the prompt box (generate gating, Enter to generate, the pickers, `@` +mentions, clear-all), the Characters dialog and the audio toolbar (toggles, Beat & Key, Tuning). ## Fonts and brand assets diff --git a/artcraft-native/src/app.rs b/artcraft-native/src/app.rs index 1cfa07a109..845fd96fe2 100644 --- a/artcraft-native/src/app.rs +++ b/artcraft-native/src/app.rs @@ -7,6 +7,7 @@ use std::time::Duration; use egui::{Id, Ui}; use serde::{Deserialize, Serialize}; +use crate::backend::audio::AudioPlayer; use crate::backend::media_cache::MediaCache; use crate::backend::wire::{Prompt, SessionUser}; use crate::backend::{Backend, Event, Modality, SHARE_URL_BASE}; @@ -14,15 +15,18 @@ use crate::feed::grid::{RatioCache, ViewMode}; use crate::feed::store::{FeedNotice, FeedStore}; use crate::feed::types::{FeedItem, MediaKind}; use crate::models::Catalog; +use crate::backend::wire::{Character, JobPhase}; use crate::overlays::account::{LoginAction, LoginDialog, SettingsAction, settings_dialog}; +use crate::overlays::characters::{CharactersAction, CharactersModal}; use crate::overlays::library_picker::{LibraryPicker, PickerAction}; use crate::overlays::lightbox::{Lightbox, LightboxAction, fit_inside}; use crate::pages::common::{self, AppRequest, Env, FeedKey}; +use crate::pages::create_audio::{AudioSettings, CreateAudioPage}; use crate::pages::create_image::{CreateImagePage, ImageSettings}; use crate::pages::create_video::{CreateVideoPage, VideoSettings}; use crate::pages::other; -use crate::prompt_box::refs::{RefKind, RefMedia}; -use crate::shell::{self, Account, Page, ShellAction}; +use crate::prompt_box::refs::{RefKind, RefMedia, References}; +use crate::shell::{self, Account, FeedToggles, Page, ShellAction}; use crate::theme; use crate::ui::toast::Toasts; use crate::ui::{creator_icons, widgets, window}; @@ -31,7 +35,7 @@ use crate::ui::{creator_icons, widgets, window}; /// borrowed mutably alongside it). macro_rules! env { ($app:ident, $ctx:expr) => { - Env { ctx: $ctx.clone(), backend: &$app.backend, cache: &mut $app.cache, toasts: &mut $app.toasts, catalog: &$app.catalog, signed_in: $app.user.is_some(), enter_to_generate: $app.enter_to_generate, view_mode: $app.view_mode, ratios: &mut $app.ratios, prompts: &$app.prompts, requests: &mut $app.requests } + Env { ctx: $ctx.clone(), backend: &$app.backend, cache: &mut $app.cache, audio: &mut $app.player, toasts: &mut $app.toasts, catalog: &$app.catalog, signed_in: $app.user.is_some(), enter_to_generate: $app.enter_to_generate, view_mode: $app.view_mode, autoplay: $app.autoplay, ratios: &mut $app.ratios, prompts: &$app.prompts, requests: &mut $app.requests, characters: &$app.characters } }; } @@ -41,6 +45,8 @@ const CREDITS_INTERVAL: f64 = 60.0; const MODEL_RETRY_INTERVAL: f64 = 15.0; /// List view: prompt records fetched per frame at most. const PROMPT_FETCHES_PER_FRAME: usize = 8; +/// Every create page's model listing. +const MODALITIES: [Modality; 3] = [Modality::Image, Modality::Video, Modality::Audio]; /// Preferences that survive restarts. #[derive(Clone, Debug, Serialize, Deserialize)] @@ -49,20 +55,23 @@ struct Saved { page: Page, enter_to_generate: bool, view_mode: ViewMode, + autoplay: bool, sidebar_open: bool, image: ImageSettings, video: VideoSettings, + audio: AudioSettings, } impl Default for Saved { fn default() -> Self { - Self { page: Page::CreateImage, enter_to_generate: false, view_mode: ViewMode::Grid, sidebar_open: true, image: ImageSettings { count: 1, ..Default::default() }, video: VideoSettings::default() } + Self { page: Page::CreateImage, enter_to_generate: false, view_mode: ViewMode::Grid, autoplay: true, sidebar_open: true, image: ImageSettings { count: 1, ..Default::default() }, video: VideoSettings::default(), audio: AudioSettings::default() } } } pub struct ArtcraftApp { backend: Backend, cache: MediaCache, + player: AudioPlayer, toasts: Toasts, catalog: Catalog, user: Option, @@ -70,9 +79,11 @@ pub struct ArtcraftApp { page: Page, enter_to_generate: bool, view_mode: ViewMode, + autoplay: bool, sidebar_open: bool, image: CreateImagePage, video: CreateVideoPage, + audio: CreateAudioPage, library: FeedStore, ratios: RatioCache, /// Prompt texts by prompt token (list view, lightbox). @@ -86,6 +97,10 @@ pub struct ArtcraftApp { lightbox: Option, picker: Option, login: Option, + characters_modal: Option, + characters: Vec, + /// Characters being made: (job token, name). + pending_characters: Vec<(String, String)>, settings_open: bool, preview: Option, requests: Vec, @@ -105,10 +120,12 @@ impl ArtcraftApp { } let backend = Backend::new(cc.egui_ctx.clone()); let cache = MediaCache::new(backend.runtime(), backend.http()); - backend.load_models(Modality::Image); - backend.load_models(Modality::Video); + let player = AudioPlayer::new(backend.runtime(), backend.http()); + for modality in MODALITIES { + backend.load_models(modality); + } backend.refresh_session(); - Self { cache, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, settings_open: false, preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend } + Self { cache, player, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, autoplay: saved.autoplay, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), audio: CreateAudioPage::new(saved.audio), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, characters_modal: None, characters: Vec::new(), pending_characters: Vec::new(), settings_open: false, preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend } } // --- Events from the backend ------------------------------------------------------------- @@ -119,13 +136,16 @@ impl ArtcraftApp { let changed = self.user.as_ref().map(|u| &u.username) != user.as_ref().map(|u| &u.username); self.user = user; if changed { - self.image.feed.reset(); - self.video.feed.reset(); - self.library.reset(); + for feed in self.feeds_mut() { + feed.reset(); + } self.next_poll = 0.0; } if self.user.is_some() { self.login = None; + if changed { + self.backend.load_characters(); + } } else { self.credits = None; if let Some(login) = &mut self.login { @@ -149,6 +169,7 @@ impl ArtcraftApp { Event::Models(modality, Ok(models)) => match modality { Modality::Image => self.catalog.set_image(models), Modality::Video => self.catalog.set_video(models), + Modality::Audio => self.catalog.set_audio(models), }, Event::Models(_, Err(message)) => { if self.models_retry_at.is_none() { @@ -157,34 +178,65 @@ impl ArtcraftApp { self.models_retry_at = Some(ctx.input(|i| i.time) + MODEL_RETRY_INTERVAL); }, Event::Cost { key, credits } => { - self.image.cost.apply(&key, credits); - self.video.cost.apply(&key, credits); + for cost in [&mut self.image.cost, &mut self.video.cost, &mut self.audio.cost] { + cost.apply(&key, credits); + } }, Event::Enqueued { modality, job_tokens, meta } => { let mut env = env!(self, ctx); match modality { Modality::Image => self.image.on_enqueued(&mut env, &job_tokens, &meta), Modality::Video => self.video.on_enqueued(&mut env, &job_tokens, &meta), + Modality::Audio => self.audio.on_enqueued(&mut env, &job_tokens, &meta), } self.next_poll = ctx.input(|i| i.time) + 2.0; }, - Event::EnqueueFailed { modality, message } => { + Event::EnqueueFailed { modality, message, status } => { let mut env = env!(self, ctx); match modality { Modality::Image => self.image.on_enqueue_failed(&mut env, &message), Modality::Video => self.video.on_enqueue_failed(&mut env, &message), + Modality::Audio => self.audio.on_enqueue_failed(&mut env, &message, status), } }, Event::Uploaded { ref_id, result } => { + if let Some(modal) = &mut self.characters_modal { + match modal.apply_upload(ref_id, result.clone()) { + Some(Err(err)) => { + self.toasts.error(format!("Failed to upload the reference image: {err}")); + return; + }, + Some(Ok(())) => return, + None => {}, + } + } if self.image.refs.find_mut(ref_id).is_some() { common::apply_upload(&mut self.toasts, &mut self.image.refs, ref_id, result); + } else if self.audio.refs.find_mut(ref_id).is_some() { + if common::apply_upload(&mut self.toasts, &mut self.audio.refs, ref_id, result) { + self.audio.check_durations(&mut self.toasts, ref_id); + } } else if common::apply_upload(&mut self.toasts, &mut self.video.refs, ref_id, result) { let mut env = env!(self, ctx); self.video.check_durations(&mut env, ref_id); } }, Event::Jobs(jobs) => { - for (kind, notices) in [(MediaKind::Image, self.image.feed.apply_jobs(MediaKind::Image, &jobs, &self.catalog)), (MediaKind::Video, self.video.feed.apply_jobs(MediaKind::Video, &jobs, &self.catalog))] { + // Characters being made finish as jobs too. + let before = self.pending_characters.len(); + let toasts = &mut self.toasts; + self.pending_characters.retain(|(job, name)| match jobs.iter().find(|j| &j.job_token == job).map(|j| j.status.phase()) { + Some(JobPhase::Succeeded) => false, + Some(JobPhase::Failed) => { + toasts.error(format!("Character \u{201c}{name}\u{201d} failed to create")); + false + }, + _ => true, + }); + if self.pending_characters.len() != before { + self.backend.load_characters(); + } + for (kind, notices) in [(MediaKind::Image, self.image.feed.apply_jobs(MediaKind::Image, &jobs, &self.catalog)), (MediaKind::Video, self.video.feed.apply_jobs(MediaKind::Video, &jobs, &self.catalog)), (MediaKind::Audio, self.audio.feed.apply_jobs(MediaKind::Audio, &jobs, &self.catalog))] { for notice in notices { match notice { FeedNotice::Completed => { @@ -200,33 +252,30 @@ impl ArtcraftApp { } } }, - Event::LibraryPage { key, page, items, total_pages } => match key.as_str() { - "picker" => { + // The picker's pages come back under its own name. + Event::LibraryPage { key, page, items, total_pages } => match FeedKey::from_name(&key) { + Some(feed) => self.feed_mut(feed).apply_library_page(feed.kind(), page, &items, total_pages), + None => { if let Some(p) = &mut self.picker { p.apply_page(page, items, total_pages); } }, - "image" => self.image.feed.apply_library_page(MediaKind::Image, page, &items, total_pages), - "video" => self.video.feed.apply_library_page(MediaKind::Video, page, &items, total_pages), - _ => self.library.apply_library_page(MediaKind::Image, page, &items, total_pages), }, Event::LibraryFailed { key, message } => { - match key.as_str() { - "picker" => { + match FeedKey::from_name(&key) { + Some(feed) => self.feed_mut(feed).library_failed(), + None => { if let Some(p) = &mut self.picker { p.loading = false; p.has_more = false; } }, - "image" => self.image.feed.library_failed(), - "video" => self.video.feed.library_failed(), - _ => self.library.library_failed(), } self.toasts.error(format!("Couldn't load your library: {message}")); }, - Event::Batch { job_token, items } => match self.batch_kinds.remove(&job_token) { - Some(MediaKind::Video) => self.video.feed.apply_batch(MediaKind::Video, &items), - _ => self.image.feed.apply_batch(MediaKind::Image, &items), + Event::Batch { job_token, items } => { + let kind = self.batch_kinds.remove(&job_token).unwrap_or(MediaKind::Image); + self.feed_mut(FeedKey::of(kind)).apply_batch(kind, &items); }, Event::Prompt(prompt) => { if let Some(text) = &prompt.maybe_positive_prompt { @@ -239,8 +288,15 @@ impl ArtcraftApp { } self.prompt_records.insert(prompt.token.clone(), prompt); }, + Event::Characters(list) => self.characters = list, + Event::CharacterCreating { job_token, name } => { + self.toasts.success(format!("Character \u{201c}{name}\u{201d} is being created")); + self.pending_characters.push((job_token, name)); + self.next_poll = 0.0; + }, + Event::CharacterChanged => self.backend.load_characters(), Event::Deleted(token) => { - for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] { + for feed in self.feeds_mut() { feed.remove_item(&token); } if self.lightbox.as_ref().is_some_and(|l| l.token == token) { @@ -274,7 +330,7 @@ impl ArtcraftApp { self.open_login(); return; }; - let refs = if page == MediaKind::Image { &self.image.refs } else { &self.video.refs }; + let refs = self.refs(page); let attached: HashSet = refs.list(kind).iter().chain(refs.first_frame.iter()).chain(refs.last_frame.iter()).filter_map(|r| r.token.clone()).collect(); let mut picker = LibraryPicker::new(kind, slot, page, max, attached); picker.loading = true; @@ -287,8 +343,7 @@ impl ArtcraftApp { }; let image = RefMedia::from_library(RefKind::Image, item.token.clone(), item.thumbnail.clone(), Some(item.full_url.clone()), 0.0); self.video.set_start_image(image); - self.lightbox = None; - self.page = Page::CreateVideo; + self.navigate(Page::CreateVideo); }, AppRequest::Recreate { kind, token } => { let Some(prompt_token) = self.find_item(&token).and_then(|i| i.prompt_token.clone()) else { @@ -314,17 +369,27 @@ impl ArtcraftApp { }, AppRequest::Download(tokens) => self.download(&tokens), AppRequest::Preview(url) => self.preview = Some(url), + AppRequest::ToggleAudio { ref_id, url } => self.player.toggle(ctx, &format!("ref:{ref_id}"), &url), + AppRequest::OpenCharacters => { + if self.user.is_none() { + self.open_login(); + return; + } + self.backend.load_characters(); + self.characters_modal = Some(CharactersModal::new()); + }, + AppRequest::PlayExternally(url) => { + if let Err(err) = open::that(&url) { + self.toasts.error(format!("Couldn't open the file: {err}")); + } + }, AppRequest::LoadMoreLibrary(key) => { let Some(user) = &self.user else { return; }; - let (feed, name) = match key { - FeedKey::Image => (&mut self.image.feed, "image"), - FeedKey::Video => (&mut self.video.feed, "video"), - FeedKey::Library => (&mut self.library, "library"), - }; - if let Some(page) = feed.next_library_page() { - self.backend.load_library(name.to_owned(), user.username.clone(), key.media_classes(), page); + let username = user.username.clone(); + if let Some(page) = self.feed_mut(key).next_library_page() { + self.backend.load_library(key.name().to_owned(), username, key.media_classes(), page); } }, } @@ -334,14 +399,15 @@ impl ArtcraftApp { match kind { MediaKind::Image => { self.image.apply_prompt(prompt); - self.page = Page::CreateImage; + self.navigate(Page::CreateImage); }, MediaKind::Video => { self.video.apply_prompt(prompt); - self.page = Page::CreateVideo; + self.navigate(Page::CreateVideo); }, + // Audio isn't recreated yet (no Recreate button offers it). + MediaKind::Audio => {}, } - self.lightbox = None; } fn download(&mut self, tokens: &[String]) { @@ -359,7 +425,7 @@ impl ArtcraftApp { for item in many { self.backend.save_url(item.full_url.clone(), dir.join(file_name_for(item))); } - for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] { + for feed in self.feeds_mut() { feed.selecting = false; feed.selected.clear(); } @@ -369,7 +435,52 @@ impl ArtcraftApp { } fn find_item(&self, token: &str) -> Option<&FeedItem> { - self.image.feed.find(token).or_else(|| self.video.feed.find(token)).or_else(|| self.library.find(token)) + self.feeds().into_iter().find_map(|f| f.find(token)) + } + + fn feeds(&self) -> [&FeedStore; 4] { + [&self.image.feed, &self.video.feed, &self.audio.feed, &self.library] + } + + fn feeds_mut(&mut self) -> [&mut FeedStore; 4] { + [&mut self.image.feed, &mut self.video.feed, &mut self.audio.feed, &mut self.library] + } + + fn feed(&self, key: FeedKey) -> &FeedStore { + match key { + FeedKey::Image => &self.image.feed, + FeedKey::Video => &self.video.feed, + FeedKey::Audio => &self.audio.feed, + FeedKey::Library => &self.library, + } + } + + fn feed_mut(&mut self, key: FeedKey) -> &mut FeedStore { + match key { + FeedKey::Image => &mut self.image.feed, + FeedKey::Video => &mut self.video.feed, + FeedKey::Audio => &mut self.audio.feed, + FeedKey::Library => &mut self.library, + } + } + + /// The references of the page that makes `kind`. + fn refs(&self, kind: MediaKind) -> &References { + match kind { + MediaKind::Image => &self.image.refs, + MediaKind::Video => &self.video.refs, + MediaKind::Audio => &self.audio.refs, + } + } + + /// Opens `page`, closing the lightbox. Leaving a page stops its audio, like the webapp's + /// players unmounting. + fn navigate(&mut self, page: Page) { + if page != self.page { + self.player.stop(); + } + self.page = page; + self.lightbox = None; } fn request_prompt(&mut self, prompt_token: Option) { @@ -386,12 +497,13 @@ impl ArtcraftApp { } } - /// The feed the top bar's toggles act on. - fn current_feed(&mut self) -> Option<&mut FeedStore> { + /// The feed the page shows (and the top bar's toggles act on). + fn page_feed(&self) -> Option { match self.page { - Page::CreateImage => Some(&mut self.image.feed), - Page::CreateVideo => Some(&mut self.video.feed), - Page::Library => Some(&mut self.library), + Page::CreateImage => Some(FeedKey::Image), + Page::CreateVideo => Some(FeedKey::Video), + Page::CreateAudio => Some(FeedKey::Audio), + Page::Library => Some(FeedKey::Library), _ => None, } } @@ -410,11 +522,10 @@ impl ArtcraftApp { } if self.models_retry_at.is_some_and(|t| now >= t) { self.models_retry_at = None; - if self.catalog.image.is_empty() { - self.backend.load_models(Modality::Image); - } - if self.catalog.video.is_empty() { - self.backend.load_models(Modality::Video); + for (modality, loaded) in MODALITIES.into_iter().zip([&self.catalog.image, &self.catalog.video, &self.catalog.audio]) { + if loaded.is_empty() { + self.backend.load_models(modality); + } } } ctx.request_repaint_after(Duration::from_secs(1)); @@ -441,10 +552,7 @@ impl ArtcraftApp { fn apply_shell_action(&mut self, action: ShellAction) { match action { - ShellAction::Navigate(page) => { - self.page = page; - self.lightbox = None; - }, + ShellAction::Navigate(page) => self.navigate(page), ShellAction::ToggleSidebar => self.sidebar_open = !self.sidebar_open, ShellAction::SignIn => self.open_login(), ShellAction::SignOut => { @@ -453,8 +561,10 @@ impl ArtcraftApp { }, ShellAction::OpenSettings => self.settings_open = true, ShellAction::SetViewMode(mode) => self.view_mode = mode, + ShellAction::SetAutoplay(on) => self.autoplay = on, ShellAction::ToggleSelect => { - if let Some(feed) = self.current_feed() { + if let Some(key) = self.page_feed() { + let feed = self.feed_mut(key); feed.selecting = !feed.selecting; feed.selected.clear(); } @@ -468,13 +578,10 @@ impl ArtcraftApp { let ctx = ui.ctx().clone(); let gap = shell::PANEL_GAP; let margin = egui::Margin { left: if self.sidebar_open { 0 } else { gap }, right: gap, top: 0, bottom: gap }; - let toggles = match self.page { - Page::CreateImage => Some((self.view_mode, self.image.feed.selecting)), - Page::CreateVideo => Some((self.view_mode, self.video.feed.selecting)), - Page::Library => Some((self.view_mode, self.library.selecting)), - _ => None, - }; + // Only feeds with videos offer the preview autoplay toggle. + let toggles = self.page_feed().map(|key| FeedToggles { mode: self.view_mode, selecting: self.feed(key).selecting, autoplay: matches!(key, FeedKey::Video | FeedKey::Library).then_some(self.autoplay) }); let mut header_action = None; + let mut open_page = None; egui::CentralPanel::default().frame(egui::Frame::NONE.fill(theme::CHROME).inner_margin(margin)).show(ui, |ui| { let panel = egui::Frame::new().fill(theme::BG).stroke(theme::hairline()).corner_radius(shell::PANEL_RADIUS); panel.show(ui, |ui| { @@ -485,13 +592,10 @@ impl ArtcraftApp { egui::CentralPanel::default().frame(egui::Frame::NONE).show(ui, |ui| { let mut env = env!(self, ctx); match self.page { - Page::Home => { - if let Some(page) = other::home(ui, env.signed_in, env.requests) { - self.page = page; - } - }, + Page::Home => open_page = other::home(ui, env.signed_in, env.requests), Page::CreateImage => self.image.ui(ui, &mut env), Page::CreateVideo => self.video.ui(ui, &mut env), + Page::CreateAudio => self.audio.ui(ui, &mut env), Page::Library => other::library(ui, &mut env, &mut self.library), page => other::coming_soon(ui, page), } @@ -501,39 +605,33 @@ impl ArtcraftApp { if let Some(action) = header_action { self.apply_shell_action(action); } - self.prefetch_list_prompts(); + if let Some(page) = open_page { + self.navigate(page); + } + self.prefetch_prompts(); } - /// The list view shows prompt texts: fetch the records for the visible feed, a few at a time. - fn prefetch_list_prompts(&mut self) { - if self.view_mode != ViewMode::List { + /// The list view and audio cards show prompt texts: fetch the records for the visible feed, a + /// few at a time. + fn prefetch_prompts(&mut self) { + let Some(key) = self.page_feed() else { return; - } - let feed = match self.page { - Page::CreateImage => &self.image.feed, - Page::CreateVideo => &self.video.feed, - Page::Library => &self.library, - _ => return, }; - let wanted: Vec = feed.items.iter().filter_map(|i| i.prompt_token.clone()).filter(|t| !self.requested_prompts.contains(t)).take(PROMPT_FETCHES_PER_FRAME).collect(); + let list = self.view_mode == ViewMode::List; + let wanted: Vec = self.feed(key).items.iter().filter(|i| list || i.kind == MediaKind::Audio).filter_map(|i| i.prompt_token.clone()).filter(|t| !self.requested_prompts.contains(t)).take(PROMPT_FETCHES_PER_FRAME).collect(); for token in wanted { self.request_prompt(Some(token)); } } fn overlays(&mut self, ctx: &egui::Context) { - if let Some(lb) = &mut self.lightbox { - let feed = match lb.kind { - _ if self.page == Page::Library => &self.library, - MediaKind::Image => &self.image.feed, - MediaKind::Video => &self.video.feed, - }; - let order = feed.ordered_tokens(); - let item = self.image.feed.find(&lb.token).or_else(|| self.video.feed.find(&lb.token)).or_else(|| self.library.find(&lb.token)).cloned(); - match item { + if let Some((kind, token)) = self.lightbox.as_ref().map(|lb| (lb.kind, lb.token.clone())) { + let key = if self.page == Page::Library { FeedKey::Library } else { FeedKey::of(kind) }; + let order = self.feed(key).ordered_tokens(); + match self.find_item(&token).cloned() { Some(item) => { let prompt = item.prompt_token.as_ref().and_then(|t| self.prompt_records.get(t)); - let actions = lb.show(ctx, &item, &order, prompt, &self.catalog, &mut self.cache); + let actions = self.lightbox.as_mut().map(|lb| lb.show(ctx, &item, &order, prompt, &self.catalog, &mut self.cache, &mut self.player)).unwrap_or_default(); for action in actions { self.lightbox_action(ctx, action, &item); } @@ -555,8 +653,11 @@ impl ArtcraftApp { }, PickerAction::Confirm(picks) => { if let Some(p) = self.picker.take() { - let refs = if p.page == MediaKind::Image { &mut self.image.refs } else { &mut self.video.refs }; - common::attach_from_library(refs, p.kind, p.slot, picks); + match p.page { + MediaKind::Image => common::attach_from_library(&mut self.image.refs, p.kind, p.slot, picks), + MediaKind::Video => common::attach_from_library(&mut self.video.refs, p.kind, p.slot, picks), + MediaKind::Audio => self.audio.attach_from_library(&mut self.toasts, &self.catalog, p.kind, picks), + } } }, } @@ -584,6 +685,27 @@ impl ArtcraftApp { None => {}, } } + if let Some(modal) = &mut self.characters_modal { + let pending: Vec = self.pending_characters.iter().map(|(_, n)| n.clone()).collect(); + for action in modal.show(ctx, &self.characters, &pending, &mut self.cache) { + match action { + CharactersAction::Close => self.characters_modal = None, + CharactersAction::Select { token, name } => { + self.video.mention_character(&name, &token); + self.characters_modal = None; + }, + CharactersAction::Upload { ref_id, path } => self.backend.upload_file(ref_id, RefKind::Image, path), + CharactersAction::Create { image_token, name, description } => self.backend.create_character(image_token, name, description), + CharactersAction::Edit { token, name, description } => self.backend.edit_character(token, name, description), + CharactersAction::Delete { token, name } => self.backend.delete_character(token, name), + CharactersAction::Preview(url) => self.preview = Some(url), + CharactersAction::Toast(message) => self.toasts.error(message), + } + if self.characters_modal.is_none() { + break; + } + } + } if self.settings_open { let username = self.user.as_ref().map(|u| u.username.clone()); match settings_dialog(ctx, &mut self.enter_to_generate, username.as_deref(), self.credits, self.backend.data_root()) { @@ -625,7 +747,7 @@ impl ArtcraftApp { if let Some(next) = self.find_item(&token).cloned() { self.request_prompt(next.prompt_token); } - for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] { + for feed in self.feeds_mut() { if feed.find(&token).is_some() { feed.last_viewed = Some(token.clone()); } @@ -663,6 +785,11 @@ impl eframe::App for ArtcraftApp { fn ui(&mut self, ui: &mut Ui, _frame: &mut eframe::Frame) { let ctx = ui.ctx().clone(); self.cache.poll(&ctx); + if let Some(err) = self.player.poll() { + self.toasts.error(err); + } + let playing = [&self.video.refs, &self.image.refs].iter().flat_map(|r| r.audios.iter()).map(|r| r.id).find(|id| self.player.is_playing(&format!("ref:{id}"))); + crate::prompt_box::deck::set_playing_audio(&ctx, playing); self.shell(ui); self.page_ui(ui); for request in std::mem::take(&mut self.requests) { @@ -677,7 +804,7 @@ impl eframe::App for ArtcraftApp { } fn save(&mut self, storage: &mut dyn eframe::Storage) { - let saved = Saved { page: self.page, enter_to_generate: self.enter_to_generate, view_mode: self.view_mode, sidebar_open: self.sidebar_open, image: self.image.settings.clone(), video: self.video.settings.clone() }; + let saved = Saved { page: self.page, enter_to_generate: self.enter_to_generate, view_mode: self.view_mode, autoplay: self.autoplay, sidebar_open: self.sidebar_open, image: self.image.settings.clone(), video: self.video.settings.clone(), audio: self.audio.settings.clone() }; eframe::set_value(storage, STORAGE_KEY, &saved); } @@ -733,6 +860,11 @@ impl ScreenshotRequest { /// `artcraft_.`, the extension taken from the URL. fn file_name_for(item: &FeedItem) -> String { - let ext = item.full_url.rsplit('.').next().filter(|e| e.len() <= 4 && e.chars().all(|c| c.is_ascii_alphanumeric())).unwrap_or(if item.kind == MediaKind::Video { "mp4" } else { "png" }); + let fallback = match item.kind { + MediaKind::Image => "png", + MediaKind::Video => "mp4", + MediaKind::Audio => "mp3", + }; + let ext = item.full_url.rsplit('.').next().filter(|e| e.len() <= 4 && e.chars().all(|c| c.is_ascii_alphanumeric())).unwrap_or(fallback); format!("artcraft_{}.{ext}", item.token) } diff --git a/artcraft-native/src/backend/audio.rs b/artcraft-native/src/backend/audio.rs new file mode 100644 index 0000000000..8b155ae0f1 --- /dev/null +++ b/artcraft-native/src/backend/audio.rs @@ -0,0 +1,343 @@ +//! Audio, one clip at a time (`audio-playback-controller`): reference cards play and stop on +//! click (the webapp's `DeckCard`), feed tracks pause, resume and seek (`WaveformAudioPlayer`). +//! Tracks also get waveforms, decoded off the UI thread. The output device opens on first use; +//! files download in the background. + +use std::collections::HashMap; +use std::io::Cursor; +use std::sync::Arc; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::time::Duration; + +use log::warn; +use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink, Source}; +use tokio::sync::Semaphore; + +/// The webapp's reference preview volume. +const PREVIEW_VOLUME: f32 = 0.2; +/// Peaks kept per waveform (resampled to however many bars are drawn). +const PEAK_COUNT: usize = 256; +/// Frames folded into one peak while decoding. +const WINDOW_FRAMES: usize = 1024; +/// Waveforms decoding at once (each downloads its whole file). +const MAX_CONCURRENT_WAVEFORMS: usize = 2; + +pub struct AudioPlayer { + rt: tokio::runtime::Handle, + http: reqwest::Client, + /// Opened on first play (and kept: closing it would stop the sound). + output: Option<(OutputStream, OutputStreamHandle)>, + sink: Option, + /// What's playing (or loading). + current: Option, + tx: Sender, + rx: Receiver, + waveforms: HashMap, + limiter: Arc, + /// The tracks' volume slider, 0–1. + track_volume: f32, +} + +/// A track's shape: peaks (0–1, loudest = 1) and its length in seconds. +#[derive(Debug, PartialEq)] +pub struct Waveform { + pub peaks: Vec, + pub duration: f32, +} + +/// What [`AudioPlayer::waveform`] knows about a track. +#[derive(Clone)] +pub enum WaveformLookup { + Loading, + Ready(Arc), + Failed, +} + +/// Where the current track is. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Playback { + /// Still downloading. + pub loading: bool, + pub paused: bool, + /// Seconds in. + pub position: f32, +} + +/// The clip that's playing, by the caller's key (a track's key is its URL). +struct Clip { + key: String, + track: bool, + /// Where to start once it has downloaded (a seek before it loaded). + start_at: f32, +} + +enum Loaded { + Clip { key: String, bytes: Result, String> }, + Waveform { url: String, waveform: Option }, +} + +impl AudioPlayer { + pub fn new(rt: tokio::runtime::Handle, http: reqwest::Client) -> Self { + let (tx, rx) = channel(); + Self { rt, http, output: None, sink: None, current: None, tx, rx, waveforms: HashMap::new(), limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_WAVEFORMS)), track_volume: 1.0 } + } + + /// Starts a reference clip under `key`, or stops it if it's what's playing. + pub fn toggle(&mut self, ctx: &egui::Context, key: &str, url: &str) { + if self.is_playing(key) { + self.stop(); + } else { + self.start(ctx, key, url, false, 0.0); + } + } + + /// Plays or pauses a feed track. + pub fn toggle_track(&mut self, ctx: &egui::Context, url: &str) { + match (&self.sink, self.is_playing(url)) { + (Some(sink), true) if sink.is_paused() => sink.play(), + (Some(sink), true) => sink.pause(), + // Still downloading: a second click cancels. + (None, true) => self.stop(), + _ => self.start(ctx, url, url, true, 0.0), + } + } + + /// Jumps a feed track to `secs` (starting it there if it isn't the current one). + pub fn seek_track(&mut self, ctx: &egui::Context, url: &str, secs: f32) { + if !self.is_playing(url) { + self.start(ctx, url, url, true, secs); + return; + } + match &self.sink { + Some(sink) => { + if let Err(err) = sink.try_seek(Duration::from_secs_f32(secs.max(0.0))) { + warn!("Audio seek failed: {err}"); + } + sink.play(); + }, + None => { + if let Some(clip) = &mut self.current { + clip.start_at = secs; + } + }, + } + } + + pub fn stop(&mut self) { + if let Some(sink) = self.sink.take() { + sink.stop(); + } + self.current = None; + } + + /// Whether `key` is playing, paused or about to play. + pub fn is_playing(&self, key: &str) -> bool { + self.current.as_ref().is_some_and(|c| c.key == key) + } + + /// Where the track at `url` is, when it's the current one. + pub fn track(&self, url: &str) -> Option { + let clip = self.current.as_ref().filter(|c| c.key == url)?; + Some(match &self.sink { + Some(sink) => Playback { loading: false, paused: sink.is_paused(), position: sink.get_pos().as_secs_f32() }, + None => Playback { loading: true, paused: false, position: clip.start_at }, + }) + } + + pub fn track_volume(&self) -> f32 { + self.track_volume + } + + pub fn set_track_volume(&mut self, volume: f32) { + self.track_volume = volume.clamp(0.0, 1.0); + if let (Some(sink), Some(clip)) = (&self.sink, &self.current) { + if clip.track { + sink.set_volume(self.track_volume); + } + } + } + + /// The waveform of the track at `url`, downloading and decoding it the first time it's asked + /// for. + pub fn waveform(&mut self, ctx: &egui::Context, url: &str) -> WaveformLookup { + if let Some(lookup) = self.waveforms.get(url) { + return lookup.clone(); + } + self.waveforms.insert(url.to_owned(), WaveformLookup::Loading); + let (http, limiter, tx, ctx, url) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone(), url.to_owned()); + self.rt.spawn(async move { + let waveform = match limiter.acquire_owned().await { + Ok(_permit) => match download(&http, &url).await { + Ok(bytes) => tokio::task::spawn_blocking(move || decode_waveform(bytes)).await.ok().flatten(), + Err(err) => { + warn!("Waveform download failed for {url}: {err}"); + None + }, + }, + Err(_) => None, + }; + let _ = tx.send(Loaded::Waveform { url, waveform }); + ctx.request_repaint(); + }); + WaveformLookup::Loading + } + + /// Starts downloads that finished and notices clips that ended. Call once per frame. Returns an + /// error message when a clip couldn't play. + pub fn poll(&mut self) -> Option { + let mut error = None; + while let Ok(loaded) = self.rx.try_recv() { + match loaded { + Loaded::Waveform { url, waveform } => { + self.waveforms.insert(url, waveform.map_or(WaveformLookup::Failed, |w| WaveformLookup::Ready(Arc::new(w)))); + }, + Loaded::Clip { key, bytes } => { + // Superseded by another clip (or stopped) while downloading. + let Some(clip) = self.current.as_ref().filter(|c| c.key == key) else { + continue; + }; + let (volume, start_at) = (if clip.track { self.track_volume } else { PREVIEW_VOLUME }, clip.start_at); + if let Err(err) = bytes.and_then(|b| self.play_bytes(b, volume, start_at)) { + warn!("Audio playback failed: {err}"); + self.current = None; + error = Some(format!("Couldn't play the audio: {err}")); + } + }, + } + } + if self.sink.as_ref().is_some_and(Sink::empty) { + self.sink = None; + self.current = None; + } + error + } + + fn start(&mut self, ctx: &egui::Context, key: &str, url: &str, track: bool, start_at: f32) { + self.stop(); + self.current = Some(Clip { key: key.to_owned(), track, start_at }); + let (http, tx, ctx, key, url) = (self.http.clone(), self.tx.clone(), ctx.clone(), key.to_owned(), url.to_owned()); + self.rt.spawn(async move { + let bytes = download(&http, &url).await; + let _ = tx.send(Loaded::Clip { key, bytes }); + ctx.request_repaint(); + }); + } + + fn play_bytes(&mut self, bytes: Arc<[u8]>, volume: f32, start_at: f32) -> Result<(), String> { + if self.output.is_none() { + self.output = Some(OutputStream::try_default().map_err(|e| e.to_string())?); + } + let Some((_, handle)) = &self.output else { + return Err("no audio output".to_owned()); + }; + let sink = Sink::try_new(handle).map_err(|e| e.to_string())?; + let source = Decoder::new(Cursor::new(bytes)).map_err(|e| e.to_string())?; + sink.set_volume(volume); + sink.append(source); + if start_at > 0.0 { + if let Err(err) = sink.try_seek(Duration::from_secs_f32(start_at)) { + warn!("Audio seek failed: {err}"); + } + } + self.sink = Some(sink); + Ok(()) + } +} + +async fn download(http: &reqwest::Client, url: &str) -> Result, String> { + let response = http.get(url).send().await.and_then(|r| r.error_for_status()).map_err(|e| e.to_string())?; + response.bytes().await.map(|b| Arc::from(b.as_ref())).map_err(|e| e.to_string()) +} + +/// Decodes a whole file into its waveform: the loudest sample of each window, folded down to +/// [`PEAK_COUNT`] peaks and scaled so the loudest is 1. +fn decode_waveform(bytes: Arc<[u8]>) -> Option { + let decoder = match Decoder::new(Cursor::new(bytes)) { + Ok(d) => d, + Err(err) => { + warn!("Waveform decode failed: {err}"); + return None; + }, + }; + let (channels, rate) = (usize::from(decoder.channels().max(1)), decoder.sample_rate().max(1)); + let window = WINDOW_FRAMES * channels; + let (mut windows, mut loudest, mut count) = (Vec::new(), 0_u16, 0_usize); + for sample in decoder { + loudest = loudest.max(sample.unsigned_abs()); + count += 1; + if count % window == 0 { + windows.push(f32::from(loudest)); + loudest = 0; + } + } + if count % window != 0 { + windows.push(f32::from(loudest)); + } + if count == 0 { + return None; + } + Some(Waveform { peaks: normalized(&resample(&windows, PEAK_COUNT)), duration: (count / channels) as f32 / rate as f32 }) +} + +/// `values` folded (by maximum) or stretched to exactly `count` entries. +pub fn resample(values: &[f32], count: usize) -> Vec { + if values.is_empty() || count == 0 { + return vec![0.0; count]; + } + (0..count) + .map(|i| { + let start = i * values.len() / count; + let end = ((i + 1) * values.len() / count).max(start + 1).min(values.len()); + values[start..end].iter().copied().fold(0.0, f32::max) + }) + .collect() +} + +fn normalized(values: &[f32]) -> Vec { + let max = values.iter().copied().fold(0.0, f32::max); + if max <= 0.0 { + return values.to_vec(); + } + values.iter().map(|v| v / max).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resampling_keeps_the_loudest_of_each_span() { + assert_eq!(resample(&[0.1, 0.9, 0.2, 0.4], 2), [0.9, 0.4]); + assert_eq!(resample(&[0.5, 1.0], 4), [0.5, 0.5, 1.0, 1.0], "short inputs stretch"); + assert_eq!(resample(&[], 3), [0.0, 0.0, 0.0]); + assert_eq!(normalized(&[0.0, 2.0, 1.0]), [0.0, 1.0, 0.5]); + } + + #[test] + fn decodes_a_wav_into_peaks_and_duration() { + // One second of 8 kHz mono: silence, then a loud half. + let rate = 8000_u32; + let samples: Vec = (0..rate).map(|i| if i < rate / 2 { 0 } else { 16000 }).collect(); + let mut wav = Vec::new(); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + samples.len() as u32 * 2).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&rate.to_le_bytes()); + wav.extend_from_slice(&(rate * 2).to_le_bytes()); + wav.extend_from_slice(&2_u16.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&(samples.len() as u32 * 2).to_le_bytes()); + for s in &samples { + wav.extend_from_slice(&s.to_le_bytes()); + } + let waveform = decode_waveform(Arc::from(wav)).expect("decodes"); + assert!((waveform.duration - 1.0).abs() < 0.01, "duration {}", waveform.duration); + assert_eq!(waveform.peaks.len(), PEAK_COUNT); + assert_eq!(waveform.peaks[0], 0.0, "starts silent"); + assert_eq!(*waveform.peaks.last().unwrap(), 1.0, "ends loud"); + } +} diff --git a/artcraft-native/src/backend/media_cache.rs b/artcraft-native/src/backend/media_cache.rs index 34bded086a..42084eee82 100644 --- a/artcraft-native/src/backend/media_cache.rs +++ b/artcraft-native/src/backend/media_cache.rs @@ -1,14 +1,18 @@ -//! Images shown in the UI: CDN thumbnails and local previews, as GPU textures. +//! Images shown in the UI: CDN thumbnails, animated video previews and local previews, as GPU +//! textures. //! //! Downloads run on the tokio runtime (a few at a time) and decode off the UI thread; the UI //! uploads finished images in [`MediaCache::poll`]. Textures nobody has drawn for a while are //! dropped once the cache grows past its budget, so long feeds don't hold every image forever. use std::collections::HashMap; +use std::io::Cursor; use std::sync::Arc; use std::sync::mpsc::{Receiver, Sender, channel}; +use std::time::Duration; use egui::{ColorImage, TextureHandle, TextureOptions}; +use image::AnimationDecoder; use log::warn; use tokio::runtime::Handle; use tokio::sync::Semaphore; @@ -17,10 +21,14 @@ use tokio::sync::Semaphore; const MAX_CONCURRENT_DOWNLOADS: usize = 6; /// Longest side of a decoded image; larger ones are downscaled before upload. const MAX_DECODED_SIDE: u32 = 2048; +/// Animated previews: longest side and frame budget (each frame is its own texture). +const ANIMATED_SIDE: u32 = 320; +const ANIMATED_MAX_FRAMES: usize = 48; /// Textures kept even when unused (roughly two screens of thumbnails). const SOFT_CAPACITY: usize = 160; -/// Frames an image may go undrawn before it can be evicted. +/// Frames an image may go undrawn before it can be evicted (animations go sooner: they're heavy). const EVICT_AFTER_FRAMES: u64 = 600; +const EVICT_ANIMATED_AFTER_FRAMES: u64 = 120; pub struct MediaCache { rt: Handle, @@ -40,14 +48,30 @@ pub enum Lookup { } enum Entry { - Loading { last_used: u64 }, - Ready { texture: TextureHandle, last_used: u64 }, + Loading { + last_used: u64, + }, + Ready { + texture: TextureHandle, + last_used: u64, + }, + /// Frames with how long each shows (seconds), and their total. + Animated { + frames: Vec<(TextureHandle, f32)>, + total: f32, + last_used: u64, + }, Failed, } struct Loaded { key: String, - image: Option, + decoded: Option, +} + +enum Decoded { + Still(ColorImage), + Frames(Vec<(ColorImage, f32)>), } impl MediaCache { @@ -58,23 +82,13 @@ impl MediaCache { /// The texture for `url`, starting its download the first time it's asked for. pub fn get(&mut self, ctx: &egui::Context, url: &str) -> Lookup { - let frame = self.frame; - match self.entries.get_mut(url) { - Some(Entry::Ready { texture, last_used }) => { - *last_used = frame; - Lookup::Ready(texture.clone()) - }, - Some(Entry::Loading { last_used }) => { - *last_used = frame; - Lookup::Loading - }, - Some(Entry::Failed) => Lookup::Failed, - None => { - self.entries.insert(url.to_owned(), Entry::Loading { last_used: frame }); - self.spawn_download(ctx, url.to_owned()); - Lookup::Loading - }, - } + self.lookup(ctx, url, url, false) + } + + /// The current frame of an animated image (a video's animated preview); still images come + /// back as they are. + pub fn get_animated(&mut self, ctx: &egui::Context, url: &str) -> Lookup { + self.lookup(ctx, &format!("anim:{url}"), url, true) } /// Decodes `bytes` off the UI thread and registers the result under `key`. @@ -85,8 +99,8 @@ impl MediaCache { self.entries.insert(key.to_owned(), Entry::Loading { last_used: self.frame }); let (tx, ctx, key) = (self.tx.clone(), ctx.clone(), key.to_owned()); self.rt.spawn_blocking(move || { - let image = decode(&bytes); - let _ = tx.send(Loaded { key, image }); + let decoded = decode(&bytes).map(Decoded::Still); + let _ = tx.send(Loaded { key, decoded }); ctx.request_repaint(); }); } @@ -94,39 +108,78 @@ impl MediaCache { /// Uploads finished downloads and evicts stale textures. Call once per frame. pub fn poll(&mut self, ctx: &egui::Context) { self.frame += 1; - while let Ok(Loaded { key, image }) = self.rx.try_recv() { + while let Ok(Loaded { key, decoded }) = self.rx.try_recv() { // Dropped while loading (evicted): don't resurrect it. let Some(entry) = self.entries.get_mut(&key) else { continue; }; - *entry = match image { - Some(image) => Entry::Ready { texture: ctx.load_texture(&key, image, TextureOptions::LINEAR), last_used: self.frame }, + *entry = match decoded { + Some(Decoded::Still(image)) => Entry::Ready { texture: ctx.load_texture(&key, image, TextureOptions::LINEAR), last_used: self.frame }, + Some(Decoded::Frames(frames)) => { + let total = frames.iter().map(|(_, d)| d).sum(); + let frames = frames.into_iter().enumerate().map(|(i, (img, d))| (ctx.load_texture(format!("{key}#{i}"), img, TextureOptions::LINEAR), d)).collect(); + Entry::Animated { frames, total, last_used: self.frame } + }, None => Entry::Failed, }; } - if self.entries.len() > SOFT_CAPACITY { - let frame = self.frame; - self.entries.retain(|_, e| match e { - Entry::Ready { last_used, .. } | Entry::Loading { last_used } => frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES, - Entry::Failed => true, - }); + let frame = self.frame; + let crowded = self.entries.len() > SOFT_CAPACITY; + self.entries.retain(|_, e| match e { + Entry::Animated { last_used, .. } => frame.saturating_sub(*last_used) < EVICT_ANIMATED_AFTER_FRAMES, + Entry::Ready { last_used, .. } | Entry::Loading { last_used } => !crowded || frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES, + Entry::Failed => true, + }); + } + + fn lookup(&mut self, ctx: &egui::Context, key: &str, url: &str, animated: bool) -> Lookup { + let frame = self.frame; + match self.entries.get_mut(key) { + Some(Entry::Ready { texture, last_used }) => { + *last_used = frame; + Lookup::Ready(texture.clone()) + }, + Some(Entry::Animated { frames, total, last_used }) => { + *last_used = frame; + let t = (ctx.input(|i| i.time) as f32).rem_euclid(total.max(0.01)); + let (mut acc, mut index) = (0.0, frames.len() - 1); + for (i, (_, d)) in frames.iter().enumerate() { + acc += d; + if t < acc { + index = i; + break; + } + } + ctx.request_repaint_after(Duration::from_secs_f32((acc - t).max(0.01))); + Lookup::Ready(frames[index].0.clone()) + }, + Some(Entry::Loading { last_used }) => { + *last_used = frame; + Lookup::Loading + }, + Some(Entry::Failed) => Lookup::Failed, + None => { + self.entries.insert(key.to_owned(), Entry::Loading { last_used: frame }); + self.spawn_download(ctx, key.to_owned(), url.to_owned(), animated); + Lookup::Loading + }, } } - fn spawn_download(&self, ctx: &egui::Context, url: String) { + fn spawn_download(&self, ctx: &egui::Context, key: String, url: String, animated: bool) { let (http, limiter, tx, ctx) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone()); self.rt.spawn(async move { - let image = match limiter.acquire_owned().await { - Ok(_permit) => download(&http, &url).await, + let decoded = match limiter.acquire_owned().await { + Ok(_permit) => download(&http, &url, animated).await, Err(_) => None, }; - let _ = tx.send(Loaded { key: url, image }); + let _ = tx.send(Loaded { key, decoded }); ctx.request_repaint(); }); } } -async fn download(http: &reqwest::Client, url: &str) -> Option { +async fn download(http: &reqwest::Client, url: &str, animated: bool) -> Option { let response = match http.get(url).send().await.and_then(|r| r.error_for_status()) { Ok(response) => response, Err(err) => { @@ -135,7 +188,7 @@ async fn download(http: &reqwest::Client, url: &str) -> Option { }, }; let bytes = response.bytes().await.ok()?; - tokio::task::spawn_blocking(move || decode(&bytes)).await.ok().flatten() + tokio::task::spawn_blocking(move || if animated { decode_frames(&bytes).map(Decoded::Frames).or_else(|| decode(&bytes).map(Decoded::Still)) } else { decode(&bytes).map(Decoded::Still) }).await.ok().flatten() } fn decode(bytes: &[u8]) -> Option { @@ -147,7 +200,45 @@ fn decode(bytes: &[u8]) -> Option { }, }; let image = if image.width().max(image.height()) > MAX_DECODED_SIDE { image.thumbnail(MAX_DECODED_SIDE, MAX_DECODED_SIDE) } else { image }; - let rgba = image.to_rgba8(); + Some(to_color_image(image.to_rgba8())) +} + +/// The frames of an animated GIF or WebP (subsampled to the budget), or `None` when it isn't +/// animated. +fn decode_frames(bytes: &[u8]) -> Option> { + let frames = match image::guess_format(bytes).ok()? { + image::ImageFormat::Gif => image::codecs::gif::GifDecoder::new(Cursor::new(bytes)).ok()?.into_frames().collect_frames().ok()?, + image::ImageFormat::WebP => { + let decoder = image::codecs::webp::WebPDecoder::new(Cursor::new(bytes)).ok()?; + if !decoder.has_animation() { + return None; + } + decoder.into_frames().collect_frames().ok()? + }, + _ => return None, + }; + if frames.len() < 2 { + return None; + } + // Keep every n-th frame, folding the skipped frames' time into the kept one. + let step = frames.len().div_ceil(ANIMATED_MAX_FRAMES); + let mut out = Vec::with_capacity(frames.len() / step + 1); + for chunk in frames.chunks(step) { + let secs: f32 = chunk + .iter() + .map(|f| { + let (n, d) = f.delay().numer_denom_ms(); + n as f32 / d.max(1) as f32 / 1000.0 + }) + .sum(); + let image = image::DynamicImage::ImageRgba8(chunk[0].buffer().clone()).thumbnail(ANIMATED_SIDE, ANIMATED_SIDE); + // Some encoders write 0 ms delays; browsers treat those as ~100 ms. + out.push((to_color_image(image.to_rgba8()), if secs < 0.02 { 0.1 * chunk.len() as f32 } else { secs })); + } + Some(out) +} + +fn to_color_image(rgba: image::RgbaImage) -> ColorImage { let size = [rgba.width() as usize, rgba.height() as usize]; - Some(ColorImage::from_rgba_unmultiplied(size, rgba.as_raw())) + ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()) } diff --git a/artcraft-native/src/backend/mod.rs b/artcraft-native/src/backend/mod.rs index 669b7a2269..c7cc136495 100644 --- a/artcraft-native/src/backend/mod.rs +++ b/artcraft-native/src/backend/mod.rs @@ -1,8 +1,10 @@ //! Everything that talks to the ArtCraft API. Calls run on a tokio runtime and report back as //! [`Event`]s, which the UI drains once per frame; nothing here blocks the UI thread. +pub mod audio; pub mod media_cache; pub mod session; +pub mod video; pub mod wire; use std::future::Future; @@ -36,7 +38,7 @@ use tokens::tokens::media_files::MediaFileToken; use crate::feed::types::MediaKind; use crate::models::{self, ModelInfo}; use session::Credentials; -use wire::{BatchMedia, MediaFile, PromptResponse, SessionInfo, SessionJobs, SessionUser, UploadResponse, UserMediaList}; +use wire::{AudioModels, BatchMedia, CharactersPage, MediaFile, PromptResponse, SessionInfo, SessionJobs, SessionUser, UploadResponse, UserMediaList}; /// Items per library page (`useGalleryData`'s `PAGE_SIZE`). pub const LIBRARY_PAGE_SIZE: u32 = 40; @@ -45,6 +47,8 @@ const MODEL_LIST_ATTEMPTS: u32 = 3; const MODEL_LIST_BACKOFF: Duration = Duration::from_millis(250); /// The share link base (`SHARE_URL_BASE`). pub const SHARE_URL_BASE: &str = "https://getartcraft.com/media/"; +/// Characters are made for this model (the Tauri app's `CharactersModal`). +const CHARACTER_MODEL: &str = "seedance_2p0"; /// Sent on every request (`artcraft_client_identity::ARTCRAFT_DESKTOP_USER_AGENT`). const USER_AGENT: &str = "storyteller-client/1.0"; @@ -53,6 +57,7 @@ const USER_AGENT: &str = "storyteller-client/1.0"; pub enum Modality { Image, Video, + Audio, } impl Modality { @@ -60,6 +65,7 @@ impl Modality { match self { Modality::Image => "image", Modality::Video => "video", + Modality::Audio => "audio", } } } @@ -69,6 +75,7 @@ impl From for Modality { match kind { MediaKind::Image => Modality::Image, MediaKind::Video => Modality::Video, + MediaKind::Audio => Modality::Audio, } } } @@ -98,6 +105,8 @@ pub enum Event { EnqueueFailed { modality: Modality, message: String, + /// The HTTP status, when the server answered. + status: Option, }, Uploaded { ref_id: u64, @@ -119,6 +128,14 @@ pub enum Event { items: Vec, }, Prompt(wire::Prompt), + /// The user's characters, newest first. + Characters(Vec), + /// A character is being made by job `job_token`. + CharacterCreating { + job_token: String, + name: String, + }, + CharacterChanged, Deleted(String), Saved(PathBuf), Toast { @@ -322,7 +339,8 @@ impl Backend { /// Fetches the OmniGen model listing for `modality`. pub fn load_models(&self, modality: Modality) { - let host = self.host.clone(); + let api = self.api(); + let host = api.host.clone(); self.spawn(move |tx| async move { let mut last_err = String::new(); for attempt in 0..MODEL_LIST_ATTEMPTS { @@ -330,15 +348,16 @@ impl Backend { tokio::time::sleep(MODEL_LIST_BACKOFF).await; } let result = match modality { - Modality::Image => omni_gen_list_image_models(OmniGenListImageModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::image_model).collect::>()), - Modality::Video => omni_gen_list_video_models(OmniGenListVideoModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::video_model).collect::>()), + Modality::Image => omni_gen_list_image_models(OmniGenListImageModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::image_model).collect::>()).map_err(|e| api_message(&e)), + Modality::Video => omni_gen_list_video_models(OmniGenListVideoModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::video_model).collect::>()).map_err(|e| api_message(&e)), + Modality::Audio => api.get_json::("/v1/omni_gen/models/audio").await.map(|r| r.models.iter().map(models::audio_model).collect::>()), }; match result { Ok(models) => { let _ = tx.send(Event::Models(modality, Ok(models))); return; }, - Err(err) => last_err = api_message(&err), + Err(err) => last_err = err, } } error!("Listing {modality:?} models failed: {last_err}"); @@ -367,7 +386,7 @@ impl Backend { let api = self.api(); self.spawn(move |tx| async move { let Some(creds) = api.creds.as_ref() else { - let _ = tx.send(Event::EnqueueFailed { modality, message: "Please sign in to generate.".to_owned() }); + let _ = tx.send(Event::EnqueueFailed { modality, message: "Please sign in to generate.".to_owned(), status: None }); return; }; fields.entry("idempotency_token").or_insert_with(|| Value::String(uuid::Uuid::new_v4().to_string())); @@ -376,14 +395,14 @@ impl Backend { Ok(response) => { let tokens = job_tokens(&response); if tokens.is_empty() { - let _ = tx.send(Event::EnqueueFailed { modality, message: "The server didn't start a job.".to_owned() }); + let _ = tx.send(Event::EnqueueFailed { modality, message: "The server didn't start a job.".to_owned(), status: None }); } else { let _ = tx.send(Event::Enqueued { modality, job_tokens: tokens, meta }); } api.send_credits(&tx).await; }, Err(err) => { - let _ = tx.send(Event::EnqueueFailed { modality, message: api_message(&err) }); + let _ = tx.send(Event::EnqueueFailed { modality, message: api_message(&err), status: api_status(&err) }); }, } }); @@ -485,6 +504,92 @@ impl Backend { }); } + /// Loads every page of the user's characters. + pub fn load_characters(&self) { + let api = self.api(); + if api.creds.is_none() { + return; + } + self.spawn(move |tx| async move { + let mut all = Vec::new(); + let mut cursor: Option = None; + loop { + let path = match cursor { + Some(c) => format!("/v1/characters/session?cursor={c}"), + None => "/v1/characters/session".to_owned(), + }; + match api.request_json::(reqwest::Method::GET, &path, None).await { + Ok(page) => { + all.extend(page.characters); + match page.next_cursor { + Some(next) if Some(next) != cursor => cursor = Some(next), + _ => break, + } + }, + Err(err) => { + warn!("Loading characters failed: {err}"); + break; + }, + } + } + let _ = tx.send(Event::Characters(all)); + }); + } + + /// Starts making a character from an uploaded reference image (`POST /v1/character/create`). + pub fn create_character(&self, image_token: String, name: String, description: Option) { + let api = self.api(); + self.spawn(move |tx| async move { + let body = serde_json::json!({ "image_media_token": image_token, "model": CHARACTER_MODEL, "uuid_idempotency_token": uuid::Uuid::new_v4().to_string(), "character_name": name, "character_description": description }); + match api.request_json::(reqwest::Method::POST, "/v1/character/create", Some(body)).await { + Ok(r) => match r.get("inference_job_token").and_then(Value::as_str) { + Some(job) => { + let _ = tx.send(Event::CharacterCreating { job_token: job.to_owned(), name }); + }, + None => { + let _ = tx.send(Event::Toast { error: true, message: "Failed to create character".to_owned() }); + }, + }, + Err(err) => { + let _ = tx.send(Event::Toast { error: true, message: if err.is_empty() { "Failed to create character".to_owned() } else { err } }); + }, + } + }); + } + + /// Renames or re-describes a character (`POST /v1/character/edit`). + pub fn edit_character(&self, token: String, name: String, description: String) { + let api = self.api(); + self.spawn(move |tx| async move { + let body = serde_json::json!({ "token": token, "updated_name": name, "updated_description": description }); + match api.request_json::(reqwest::Method::POST, "/v1/character/edit", Some(body)).await { + Ok(_) => { + let _ = tx.send(Event::Toast { error: false, message: "Character updated".to_owned() }); + let _ = tx.send(Event::CharacterChanged); + }, + Err(err) => { + let _ = tx.send(Event::Toast { error: true, message: format!("Failed to update character: {err}") }); + }, + } + }); + } + + /// `DELETE /v1/character/{token}`. + pub fn delete_character(&self, token: String, name: String) { + let api = self.api(); + self.spawn(move |tx| async move { + match api.request_json::(reqwest::Method::DELETE, &format!("/v1/character/{token}"), None).await { + Ok(_) => { + let _ = tx.send(Event::Toast { error: false, message: format!("Character \"{name}\" deleted") }); + let _ = tx.send(Event::CharacterChanged); + }, + Err(err) => { + let _ = tx.send(Event::Toast { error: true, message: format!("Failed to delete character: {err}") }); + }, + } + }); + } + pub fn delete_media(&self, token: String) { let api = self.api(); self.spawn(move |tx| async move { @@ -589,8 +694,16 @@ struct Api { impl Api { /// A GET with the desktop app's identity headers and cookies, decoded leniently. async fn get_json(&self, path: &str) -> Result { + self.request_json(reqwest::Method::GET, path, None).await + } + + /// Any JSON call with the desktop app's identity headers and cookies, decoded leniently. + async fn request_json(&self, method: reqwest::Method, path: &str, body: Option) -> Result { let url = format!("{}{path}", self.host.to_api_hostname_and_scheme()); - let mut request = self.http.get(&url).header("Accept", "application/json").header("Origin", self.host.request_origin()); + let mut request = self.http.request(method, &url).header("Accept", "application/json").header("Origin", self.host.request_origin()); + if let Some(body) = body { + request = request.json(&body); + } if let Some(cookie) = self.creds.as_ref().and_then(StorytellerCredentialSet::maybe_as_cookie_header) { request = request.header("Cookie", cookie); } @@ -644,7 +757,7 @@ impl Api { } let info = self.get_json::(&format!("/v1/media_files/file/{token}")).await.ok(); let file = info.map(|r| r.media_file); - UploadedMedia { duration_secs: file.as_ref().and_then(|f| f.maybe_duration_millis).map_or(0.0, |ms| ms as f32 / 1000.0), thumbnail: file.as_ref().and_then(|f| f.media_links.thumbnail(256)), full_url: file.as_ref().map(|f| f.media_links.cdn_url.clone()).filter(|u| !u.is_empty()), token } + UploadedMedia { duration_secs: file.as_ref().and_then(|f| f.maybe_duration_millis).map_or(0.0, |ms| ms as f32 / 1000.0), thumbnail: file.as_ref().and_then(|f| f.thumbnail(256)), full_url: file.as_ref().map(|f| f.media_links.cdn_url.clone()).filter(|u| !u.is_empty()), token } } } @@ -687,6 +800,24 @@ pub fn api_message(err: &StorytellerError) -> String { } } +/// The HTTP status behind a client error, when there was a response. +fn api_status(err: &StorytellerError) -> Option { + let StorytellerError::Api(api) = err else { + return None; + }; + Some(match api { + ApiError::InvalidRequest(_) => 400, + ApiError::Unauthorized(_) => 401, + ApiError::PaymentRequired(_) => 402, + ApiError::Forbidden(_) => 403, + ApiError::NotFound(_) => 404, + ApiError::TooManyRequests(_) => 429, + ApiError::InternalServerError { .. } => 500, + ApiError::UncategorizedBadResponseWithStatusAndBody { status_code, .. } => status_code.as_u16(), + _ => return None, + }) +} + fn api_error_message(err: &ApiError) -> String { let body = match err { ApiError::InvalidRequest(b) | ApiError::Unauthorized(b) | ApiError::PaymentRequired(b) | ApiError::Forbidden(b) | ApiError::NotFound(b) | ApiError::TooManyRequests(b) => Some(b.as_str()), diff --git a/artcraft-native/src/backend/video.rs b/artcraft-native/src/backend/video.rs new file mode 100644 index 0000000000..a7f3d61b0e --- /dev/null +++ b/artcraft-native/src/backend/video.rs @@ -0,0 +1,422 @@ +//! In-app video playback through FFmpeg (when it's installed): `ffprobe` reads the clip's size, +//! rate and length, one `ffmpeg` streams RGBA frames into a texture and another streams PCM into +//! rodio. Pausing and seeking restart both at the new position. Without FFmpeg the lightbox opens +//! videos in the system player instead. + +use std::collections::VecDeque; +use std::io::Read; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, channel, sync_channel}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use egui::{Align2, Color32, ColorImage, FontId, Rect, Sense, TextureHandle, TextureOptions, Ui, pos2, vec2}; +use log::warn; +use rodio::{OutputStream, OutputStreamHandle, Sink, Source}; + +use crate::theme; +use crate::ui::icons::{self, Icon}; + +/// Frames are scaled down to at most this width (the lightbox never shows them larger). +const MAX_WIDTH: u32 = 1280; +const AUDIO_RATE: u32 = 44_100; +/// Decoded frames buffered ahead of the clock (back-pressure on `ffmpeg`). +const FRAME_QUEUE: usize = 4; +const CONTROLS_HEIGHT: f32 = 40.0; + +/// Whether `ffmpeg` and `ffprobe` are on the PATH (checked once). +pub fn ffmpeg_available() -> bool { + static AVAILABLE: OnceLock = OnceLock::new(); + *AVAILABLE.get_or_init(|| ["ffmpeg", "ffprobe"].iter().all(|tool| command(tool).arg("-version").stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()))) +} + +#[derive(Clone, Copy, Debug)] +struct VideoInfo { + width: u32, + height: u32, + fps: f64, + duration: f64, +} + +pub struct VideoPlayer { + url: String, + probe: Option>>, + info: Option, + stream: Option, + texture: Option, + /// Media time of the frame on screen. + position: f64, + muted: bool, + error: Option, + output: Option<(OutputStream, OutputStreamHandle)>, +} + +/// One run of the decoders from a start position. +struct Stream { + start: f64, + started: Instant, + frames: Receiver<(ColorImage, f64)>, + next: Option<(ColorImage, f64)>, + ended: bool, + children: Vec, + sink: Option, +} + +impl Drop for Stream { + fn drop(&mut self) { + for child in &mut self.children { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +impl VideoPlayer { + /// Starts probing `url`; playback begins as soon as its size is known. + pub fn new(url: &str) -> Self { + let (tx, rx) = channel(); + let probe_url = url.to_owned(); + std::thread::spawn(move || { + let _ = tx.send(probe(&probe_url)); + }); + Self { url: url.to_owned(), probe: Some(rx), info: None, stream: None, texture: None, position: 0.0, muted: false, error: None, output: None } + } + + pub fn url(&self) -> &str { + &self.url + } + + /// Draws the video fitted inside `rect` with its controls; `poster` (the still frame) shows + /// until the first decoded frame arrives. + pub fn ui(&mut self, ui: &mut Ui, rect: Rect, poster: Option<&TextureHandle>) { + self.poll_probe(); + let ctx = ui.ctx().clone(); + self.advance(&ctx); + ui.painter().rect_filled(rect, 0.0, Color32::BLACK); + let video_area = Rect::from_min_max(rect.min, pos2(rect.right(), rect.bottom() - CONTROLS_HEIGHT)); + match (&self.texture, &self.error) { + (_, Some(err)) => { + ui.painter().text(video_area.center(), Align2::CENTER_CENTER, format!("Couldn't play this video: {err}"), FontId::new(13.0, egui::FontFamily::Proportional), theme::BAD); + }, + (Some(texture), None) => { + let fit = crate::overlays::lightbox::fit_inside(texture.size_vec2(), video_area.shrink(4.0).size()); + egui::Image::new(texture).paint_at(ui, Rect::from_center_size(video_area.center(), fit)); + }, + (None, None) => { + if let Some(poster) = poster { + let fit = crate::overlays::lightbox::fit_inside(poster.size_vec2(), video_area.shrink(4.0).size()); + egui::Image::new(poster).tint(Color32::from_gray(150)).paint_at(ui, Rect::from_center_size(video_area.center(), fit)); + } + icons::paint_spinner(ui, Rect::from_center_size(video_area.center(), vec2(32.0, 32.0)), Color32::WHITE); + }, + } + // Clicking the picture toggles play/pause. + let click = ui.interact(video_area, ui.id().with(("video-surface", &self.url)), Sense::click()); + if click.clicked() { + self.toggle_pause(); + } + self.controls(ui, Rect::from_min_max(pos2(rect.left(), rect.bottom() - CONTROLS_HEIGHT), rect.max)); + if self.stream.is_some() { + ctx.request_repaint_after(Duration::from_millis(10)); + } + } + + fn controls(&mut self, ui: &mut Ui, bar: Rect) { + let p = ui.painter().clone(); + p.rect_filled(bar, 0.0, theme::fade(theme::PANEL, 0.95)); + let playing = self.stream.is_some(); + let button = Rect::from_center_size(pos2(bar.left() + 22.0, bar.center().y), vec2(28.0, 28.0)); + let resp = ui.interact(button, ui.id().with(("video-play", &self.url)), Sense::click()).on_hover_text(if playing { "Pause" } else { "Play" }).on_hover_cursor(egui::CursorIcon::PointingHand); + if playing { + for dx in [-3.5, 3.5] { + p.rect_filled(Rect::from_center_size(button.center() + vec2(dx, 0.0), vec2(3.0, 12.0)), 0.0, theme::INK); + } + } else { + icons::paint(&p, button.shrink(7.0), Icon::Play, theme::INK); + } + if resp.clicked() { + self.toggle_pause(); + } + let duration = self.info.map_or(0.0, |i| i.duration); + let time = format!("{} / {}", clock(self.position), clock(duration)); + let label = p.text(pos2(button.right() + 8.0, bar.center().y), Align2::LEFT_CENTER, time, FontId::new(11.5, egui::FontFamily::Monospace), theme::MUTED); + let mute = Rect::from_center_size(pos2(bar.right() - 22.0, bar.center().y), vec2(28.0, 28.0)); + let mute_resp = ui.interact(mute, ui.id().with(("video-mute", &self.url)), Sense::click()).on_hover_text(if self.muted { "Unmute" } else { "Mute" }).on_hover_cursor(egui::CursorIcon::PointingHand); + icons::paint(&p, mute.shrink(7.0), if self.muted { Icon::VolumeOff } else { Icon::Volume }, theme::INK); + if mute_resp.clicked() { + self.muted = !self.muted; + if let Some(sink) = self.stream.as_ref().and_then(|s| s.sink.as_ref()) { + sink.set_volume(if self.muted { 0.0 } else { 1.0 }); + } + } + // The scrubber: click or drag to seek. + let track = Rect::from_min_max(pos2(label.right() + 14.0, bar.center().y - 8.0), pos2(mute.left() - 12.0, bar.center().y + 8.0)); + let scrub = ui.interact(track, ui.id().with(("video-scrub", &self.url)), Sense::click_and_drag()).on_hover_cursor(egui::CursorIcon::PointingHand); + let line = Rect::from_center_size(track.center(), vec2(track.width(), 4.0)); + p.rect_filled(line, egui::CornerRadius::same(2), theme::WASH_HOVER); + let fraction = if duration > 0.0 { (self.position / duration).clamp(0.0, 1.0) as f32 } else { 0.0 }; + p.rect_filled(Rect::from_min_size(line.min, vec2(line.width() * fraction, line.height())), egui::CornerRadius::same(2), theme::ACCENT_400); + if (scrub.clicked() || scrub.drag_stopped()) && duration > 0.0 { + if let Some(pos) = scrub.interact_pointer_pos() { + let t = (((pos.x - line.left()) / line.width()).clamp(0.0, 1.0) as f64) * duration; + self.seek(t); + } + } + } + + fn poll_probe(&mut self) { + let Some(rx) = &self.probe else { + return; + }; + match rx.try_recv() { + Ok(Ok(info)) => { + self.info = Some(info); + self.probe = None; + self.play_from(0.0); + }, + Ok(Err(err)) => { + self.error = Some(err); + self.probe = None; + }, + Err(_) => {}, + } + } + + /// Shows the newest frame that's due; loops at the end. + fn advance(&mut self, ctx: &egui::Context) { + let Some(stream) = &mut self.stream else { + return; + }; + let now = stream.start + stream.started.elapsed().as_secs_f64(); + let mut shown = None; + loop { + if stream.next.is_none() { + match stream.frames.try_recv() { + Ok(frame) => stream.next = Some(frame), + Err(TryRecvError::Disconnected) => { + stream.ended = true; + break; + }, + Err(TryRecvError::Empty) => break, + } + } + match &stream.next { + Some((_, pts)) if *pts <= now => shown = stream.next.take(), + _ => break, + } + } + if let Some((image, pts)) = shown { + self.position = pts; + match &mut self.texture { + Some(t) => t.set(image, TextureOptions::LINEAR), + None => self.texture = Some(ctx.load_texture(format!("video:{}", self.url), image, TextureOptions::LINEAR)), + } + } + if stream.ended && stream.next.is_none() { + self.play_from(0.0); + } + } + + fn toggle_pause(&mut self) { + if self.stream.take().is_none() && self.info.is_some() { + self.play_from(self.position); + } + } + + fn seek(&mut self, t: f64) { + self.position = t; + self.play_from(t); + } + + fn play_from(&mut self, start: f64) { + self.stream = None; + let Some(info) = self.info else { + return; + }; + match self.spawn(info, start) { + Ok(stream) => self.stream = Some(stream), + Err(err) => self.error = Some(err), + } + } + + fn spawn(&mut self, info: VideoInfo, start: f64) -> Result { + let (w, h) = output_size(info); + let mut video = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{start:.3}"), "-i", &self.url, "-an", "-vf", &format!("scale={w}:{h}"), "-f", "rawvideo", "-pix_fmt", "rgba", "-"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(|e| format!("ffmpeg: {e}"))?; + let stdout = video.stdout.take().ok_or("ffmpeg has no output")?; + let (frame_tx, frame_rx) = sync_channel(FRAME_QUEUE); + let fps = info.fps.max(1.0); + std::thread::spawn(move || read_frames(stdout, w, h, start, fps, frame_tx)); + let mut children = vec![video]; + let sink = self.spawn_audio(start, &mut children); + Ok(Stream { start, started: Instant::now(), frames: frame_rx, next: None, ended: false, children, sink }) + } + + /// The soundtrack as PCM into rodio (silently skipped when the clip has none). + fn spawn_audio(&mut self, start: f64, children: &mut Vec) -> Option { + let mut audio = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{start:.3}"), "-i", &self.url, "-vn", "-ac", "2", "-ar", &AUDIO_RATE.to_string(), "-f", "s16le", "-"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().ok()?; + let mut stdout = audio.stdout.take()?; + children.push(audio); + if self.output.is_none() { + self.output = OutputStream::try_default().map_err(|e| warn!("No audio output: {e}")).ok(); + } + let (_, handle) = self.output.as_ref()?; + let sink = Sink::try_new(handle).ok()?; + let (tx, rx) = channel::>(); + std::thread::spawn(move || { + let mut buf = vec![0u8; 8192]; + while let Ok(n) = stdout.read(&mut buf) { + if n == 0 || tx.send(buf[..n - n % 2].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()).is_err() { + break; + } + } + }); + sink.set_volume(if self.muted { 0.0 } else { 1.0 }); + sink.append(PcmStream { rx, buf: VecDeque::new() }); + Some(sink) + } +} + +/// Reads fixed-size RGBA frames from `ffmpeg` and stamps them with their media time. +fn read_frames(mut stdout: impl Read, w: u32, h: u32, start: f64, fps: f64, tx: SyncSender<(ColorImage, f64)>) { + let mut buf = vec![0u8; (w * h * 4) as usize]; + let mut n = 0u64; + while stdout.read_exact(&mut buf).is_ok() { + let image = ColorImage::from_rgba_unmultiplied([w as usize, h as usize], &buf); + if tx.send((image, start + n as f64 / fps)).is_err() { + break; + } + n += 1; + } +} + +/// Live PCM from a channel; silence while it waits, the end when the sender's gone. +struct PcmStream { + rx: Receiver>, + buf: VecDeque, +} + +impl Iterator for PcmStream { + type Item = i16; + + fn next(&mut self) -> Option { + if self.buf.is_empty() { + match self.rx.try_recv() { + Ok(chunk) => self.buf.extend(chunk), + Err(TryRecvError::Empty) => return Some(0), + Err(TryRecvError::Disconnected) => return None, + } + } + self.buf.pop_front().or(Some(0)) + } +} + +impl Source for PcmStream { + fn current_frame_len(&self) -> Option { + None + } + + fn channels(&self) -> u16 { + 2 + } + + fn sample_rate(&self) -> u32 { + AUDIO_RATE + } + + fn total_duration(&self) -> Option { + None + } +} + +/// Reads the clip's size, rate and length. A small probe keeps this ~2 s on CDN files instead of +/// ~14 s (ffprobe otherwise scans much of the file over HTTP). +fn probe(url: &str) -> Result { + #[derive(serde::Deserialize, Default)] + #[serde(default)] + struct Probe { + streams: Vec, + format: ProbeFormat, + } + #[derive(serde::Deserialize, Default)] + #[serde(default)] + struct ProbeStream { + width: u32, + height: u32, + avg_frame_rate: String, + } + #[derive(serde::Deserialize, Default)] + #[serde(default)] + struct ProbeFormat { + duration: String, + } + let out = command("ffprobe").args(["-v", "error", "-probesize", "65536", "-analyzeduration", "0", "-select_streams", "v:0", "-show_entries", "stream=width,height,avg_frame_rate:format=duration", "-of", "json", url]).output().map_err(|e| format!("ffprobe: {e}"))?; + let probe: Probe = serde_json::from_slice(&out.stdout).map_err(|e| format!("ffprobe: {e}"))?; + let stream = probe.streams.first().ok_or("no video stream")?; + let fps = match stream.avg_frame_rate.split_once('/') { + Some((n, d)) => n.parse::().unwrap_or(24.0) / d.parse::().unwrap_or(1.0).max(1.0), + None => stream.avg_frame_rate.parse().unwrap_or(24.0), + }; + Ok(VideoInfo { width: stream.width.max(2), height: stream.height.max(2), fps: if fps.is_finite() && fps > 0.0 { fps } else { 24.0 }, duration: probe.format.duration.parse().unwrap_or(0.0) }) +} + +/// The decoded size: at most `MAX_WIDTH` wide, both sides even (as `scale` requires). +fn output_size(info: VideoInfo) -> (u32, u32) { + let w = info.width.min(MAX_WIDTH) & !1; + let h = ((info.height as f64 * w as f64 / info.width as f64).round() as u32).max(2) & !1; + (w, h) +} + +/// `m:ss`. +fn clock(secs: f64) -> String { + let s = secs.max(0.0) as u64; + format!("{}:{:02}", s / 60, s % 60) +} + +/// A command that doesn't flash a console window on Windows. +fn command(program: &str) -> Command { + #[allow(unused_mut)] + let mut cmd = Command::new(program); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + cmd.creation_flags(CREATE_NO_WINDOW); + } + cmd +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn output_sizes_are_even_and_capped() { + let info = |width, height| VideoInfo { width, height, fps: 24.0, duration: 5.0 }; + assert_eq!(output_size(info(1920, 1080)), (1280, 720)); + assert_eq!(output_size(info(721, 1281)), (720, 1278)); + assert_eq!(clock(65.9), "1:05"); + } + + /// Needs FFmpeg on the PATH: `cargo test -p artcraft_native -- --ignored`. + #[test] + #[ignore] + fn probes_and_decodes_a_generated_clip() { + let dir = std::env::temp_dir().join("artcraft-native-video-test"); + std::fs::create_dir_all(&dir).unwrap(); + let clip = dir.join("clip.mp4"); + let status = command("ffmpeg").args(["-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc=duration=2:size=640x360:rate=24", "-f", "lavfi", "-i", "sine=duration=2", "-shortest", "-pix_fmt", "yuv420p"]).arg(&clip).status().unwrap(); + assert!(status.success()); + let url = clip.to_string_lossy().into_owned(); + let info = probe(&url).unwrap(); + assert_eq!((info.width, info.height), (640, 360)); + assert!((info.fps - 24.0).abs() < 0.01 && (info.duration - 2.0).abs() < 0.2); + + let mut player = VideoPlayer { url: url.clone(), probe: None, info: Some(info), stream: None, texture: None, position: 0.0, muted: true, error: None, output: None }; + let stream = player.spawn(info, 1.0).unwrap(); + let (frame, pts) = stream.frames.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(frame.size, [640, 360]); + assert!((pts - 1.0).abs() < 1e-9, "frames are stamped from the start position"); + let count = 1 + stream.frames.iter().count(); + assert!((20..=28).contains(&count), "about a second of frames at 24 fps, got {count}"); + } +} diff --git a/artcraft-native/src/backend/wire.rs b/artcraft-native/src/backend/wire.rs index d02490cf52..80c7332aa6 100644 --- a/artcraft-native/src/backend/wire.rs +++ b/artcraft-native/src/backend/wire.rs @@ -42,6 +42,17 @@ impl MediaLinks { } } +impl MediaLinks { + /// A video's animated preview, `width` pixels wide where the CDN can resize. + pub fn animated_preview(&self, width: u32) -> Option { + let previews = self.maybe_video_previews.as_ref()?; + if !previews.animated_thumbnail_template.is_empty() { + return Some(previews.animated_thumbnail_template.replace("{WIDTH}", &width.to_string())); + } + (!previews.animated.is_empty()).then(|| previews.animated.clone()) + } +} + /// `GET /v1/media_files/list/user/{username}`. #[derive(Debug, Default, Deserialize)] #[serde(default)] @@ -70,10 +81,56 @@ pub struct MediaFile { pub maybe_model_type: Option, pub maybe_origin_model_type: Option, pub maybe_duration_millis: Option, + pub maybe_title: Option, pub media_links: MediaLinks, pub created_at: String, } +impl MediaFile { + /// The file's still thumbnail; audio files have none (their CDN link is the audio itself). + pub fn thumbnail(&self, width: u32) -> Option { + if self.media_class.as_deref() == Some("audio") { + return None; + } + self.media_links.thumbnail(width) + } +} + +/// `GET /v1/omni_gen/models/audio`. The client has no binding for it (and the API's type only +/// serializes), so it's read here; capability flags are left out when false. +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +pub struct AudioModels { + pub success: bool, + pub models: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct AudioModel { + pub model: String, + pub model_creator: Option, + pub full_name: Option, + pub extra_info_short: Option, + pub text_prompt_supported: Option, + pub style_prompt_supported: bool, + pub audio_references_supported: bool, + pub audio_references_max: Option, + pub image_references_supported: bool, + pub image_references_max: Option, + pub keep_lyrics_supported: bool, + pub instrumental_toggle_supported: bool, + pub loopable_toggle_supported: bool, + pub bpm_supported: bool, + pub musical_key_supported: bool, + pub sample_rate_hz_options: Vec, + pub sample_rate_hz_default: Option, + pub speed_supported: bool, + pub volume_supported: bool, + pub pitch_supported: bool, + pub is_disabled: bool, +} + /// `GET /v1/media_files/batch_gen_redux/{token}`. #[derive(Debug, Default, Deserialize)] #[serde(default)] @@ -194,6 +251,36 @@ pub struct ContextImage { pub media_links: MediaLinks, } +/// A saved character (`GET /v1/characters/session`). +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct Character { + pub token: String, + pub name: String, + pub maybe_description: Option, + pub maybe_avatar: Option, + pub maybe_full_image: Option, + pub models: Vec, +} + +impl Character { + pub fn avatar_url(&self) -> Option { + self.maybe_avatar.as_ref().or(self.maybe_full_image.as_ref()).and_then(|l| l.thumbnail(256)) + } + + pub fn full_url(&self) -> Option { + self.maybe_full_image.as_ref().or(self.maybe_avatar.as_ref()).map(|l| l.cdn_url.clone()).filter(|u| !u.is_empty()) + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +pub struct CharactersPage { + pub success: bool, + pub characters: Vec, + pub next_cursor: Option, +} + /// `GET /v1/session` (only the parts the app shows). #[derive(Debug, Default, Deserialize)] #[serde(default)] diff --git a/artcraft-native/src/feed/grid.rs b/artcraft-native/src/feed/grid.rs index 2cbf2bf59d..97fbceba1c 100644 --- a/artcraft-native/src/feed/grid.rs +++ b/artcraft-native/src/feed/grid.rs @@ -1,16 +1,17 @@ //! The generation feed as a masonry grid (`GenerationGridView`) or a list (`GenerationListView`): //! running jobs first-class alongside failures and finished media, newest first, loading more as -//! the user scrolls. +//! the user scrolls. Audio plays in place, with its waveform. use std::collections::{HashMap, HashSet}; use egui::{Align2, Color32, CornerRadius, FontFamily, FontId, Id, Mesh, Rect, Sense, Stroke, Ui, pos2, vec2}; +use crate::backend::audio::AudioPlayer; use crate::backend::media_cache::{Lookup, MediaCache}; use crate::feed::types::{FailedJob, FeedItem, MediaKind, PendingJob, pending_status, time_ago}; use crate::theme; use crate::ui::icons::{self, Icon}; -use crate::ui::{creator_icons, widgets}; +use crate::ui::{creator_icons, waveform, widgets}; /// `react-masonry-css` gutter and the cap on tall portraits (≈ 5:7). const GAP: f32 = 8.0; @@ -18,6 +19,8 @@ const MAX_RATIO: f32 = 1.4; /// Start loading the next page this far before the end. const LOAD_MORE_MARGIN: f32 = 400.0; const LIST_THUMB: f32 = 100.0; +/// The list's compact audio player is at most `max-w-md` wide. +const LIST_PLAYER_MAX: f32 = 448.0; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum ViewMode { @@ -61,6 +64,8 @@ pub struct FeedView<'a> { /// Prompt texts by prompt token (list view and pending cards). pub prompts: &'a HashMap, pub make_video: bool, + /// Play videos' animated previews instead of their still frames. + pub autoplay: bool, pub bottom_padding: f32, } @@ -84,7 +89,7 @@ impl Entry<'_> { #[derive(Clone, Default)] pub struct RatioCache(HashMap); -pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &mut RatioCache, names: &dyn ModelNames) -> Vec { +pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &mut RatioCache, audio: &mut AudioPlayer, names: &dyn ModelNames) -> Vec { let mut entries: Vec> = view.pending.iter().map(Entry::Pending).chain(view.failed.iter().map(Entry::Failed)).chain(view.items.iter().map(Entry::Item)).collect(); entries.sort_by_key(|e| std::cmp::Reverse(e.time())); let mut actions = Vec::new(); @@ -111,10 +116,10 @@ pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &m match (entry, view.mode) { (Entry::Pending(job), ViewMode::Grid) => pending_card(ui, rect, job, now, names), (Entry::Failed(job), ViewMode::Grid) => failed_card(ui, view.id, rect, job, cache, names, &mut actions), - (Entry::Item(item), ViewMode::Grid) => item_card(ui, view, rect, item, cache, ratios, names, &mut actions), + (Entry::Item(item), ViewMode::Grid) => item_card(ui, view, rect, item, cache, ratios, audio, names, &mut actions), (Entry::Pending(job), ViewMode::List) => pending_row(ui, rect, job, now, names), (Entry::Failed(job), ViewMode::List) => failed_row(ui, view.id, rect, job, names, &mut actions), - (Entry::Item(item), ViewMode::List) => item_row(ui, view, rect, item, cache, names, now, &mut actions), + (Entry::Item(item), ViewMode::List) => item_row(ui, view, rect, item, cache, audio, names, now, &mut actions), } } @@ -226,25 +231,15 @@ fn failed_card(ui: &mut Ui, id: Id, rect: Rect, job: &FailedJob, cache: &mut Med } #[allow(clippy::too_many_arguments)] -fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache, names: &dyn ModelNames, actions: &mut Vec) { +fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache, audio: &mut AudioPlayer, names: &dyn ModelNames, actions: &mut Vec) { let resp = ui.interact(rect, view.id.with(("item", &item.token)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand); - let hovered = resp.hovered(); - ui.painter().rect_filled(rect, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4)); - match item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) { - Some(Lookup::Ready(t)) => { - let size = t.size_vec2(); - if size.x > 0.0 && !ratios.0.contains_key(&item.token) { - ratios.0.insert(item.token.clone(), size.y / size.x); - } - egui::Image::new(&t).uv(cover_uv_for(size, rect.size())).corner_radius(theme::RADIUS).paint_at(ui, rect); - }, - Some(Lookup::Loading) => shimmer(ui, rect), - _ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), if item.kind == MediaKind::Video { Icon::Video } else { Icon::Image }, theme::FAINT), - } - if item.kind == MediaKind::Video { - let badge = Rect::from_center_size(rect.center(), vec2(40.0, 40.0)); - ui.painter().circle_filled(badge.center(), 20.0, Color32::from_black_alpha(110)); - icons::paint(ui.painter(), badge.shrink(12.0).translate(vec2(1.5, 0.0)), Icon::Play, theme::fade(Color32::WHITE, 0.9)); + // The pointer may be on an action button (or the audio player) rather than the card itself. + let hovered = resp.hovered() || ui.rect_contains_pointer(rect); + let is_audio = item.kind == MediaKind::Audio; + if is_audio { + audio_face(ui, view, rect, item, audio); + } else { + visual_face(ui, view, rect, item, cache, ratios); } let selected = view.selected.contains(&item.token); let ring = if selected { @@ -276,25 +271,17 @@ fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cach p.galley(pos2(badge.left() + 20.0, badge.center().y - galley.size().y / 2.0), galley, Color32::WHITE); } let mut action_clicked = false; - // The pointer may be on an action button rather than the card itself. - if (hovered || ui.rect_contains_pointer(rect)) && !view.selecting { - let footer = bottom_gradient(ui, rect, 56.0, 178); - let chip_y = footer.bottom() - 18.0; - let mut x = footer.left() + 8.0; - x = chip(ui, pos2(x, chip_y), Some(if item.kind == MediaKind::Video { Icon::Video } else { Icon::Image }), None, item.kind.label()); + if hovered && !view.selecting { + // Audio keeps its player at the bottom, so its chips and actions go on top. + let band = edge_fade(ui, rect, 56.0, 178, is_audio); + let chip_y = if is_audio { band.top() + 18.0 } else { band.bottom() - 18.0 }; + let mut x = band.left() + 8.0; + x = chip(ui, pos2(x, chip_y), Some(kind_icon(item.kind)), None, item.kind.label()); if let Some(model) = item.model_id.as_deref() { chip(ui, pos2(x + 6.0, chip_y), None, Some(&names.creator(model)), &truncate(&names.display_name(model), 16)); } - let mut buttons: Vec<(Icon, &str, FeedAction)> = Vec::new(); - if item.prompt_token.is_some() { - buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone()))); - } - if view.make_video && item.kind == MediaKind::Image { - buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone()))); - } - buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone()))); - buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone()))); - let pill = Rect::from_min_size(pos2(footer.right() - 8.0 - 8.0 - 28.0 * buttons.len() as f32, chip_y - 18.0), vec2(8.0 + 28.0 * buttons.len() as f32, 36.0)); + let buttons = item_actions(view, item, None); + let pill = Rect::from_min_size(pos2(band.right() - 8.0 - 8.0 - 28.0 * buttons.len() as f32, chip_y - 18.0), vec2(8.0 + 28.0 * buttons.len() as f32, 36.0)); ui.painter().rect_filled(pill, 0.0, Color32::from_black_alpha(153)); let mut child = ui.new_child(egui::UiBuilder::new().max_rect(pill.shrink(4.0)).layout(egui::Layout::left_to_right(egui::Align::Center))); child.spacing_mut().item_spacing.x = 0.0; @@ -310,6 +297,87 @@ fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cach } } +/// An image or video card's face: the thumbnail (or animated preview) and a video's play badge. +fn visual_face(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache) { + ui.painter().rect_filled(rect, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4)); + // Videos play their animated preview when previews are on (the still shows while it loads). + let animated = item.animated.as_deref().filter(|_| view.autoplay && item.kind == MediaKind::Video).map(|u| cache.get_animated(ui.ctx(), u)); + let lookup = match animated { + Some(Lookup::Ready(t)) => Some(Lookup::Ready(t)), + _ => item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)), + }; + match lookup { + Some(Lookup::Ready(t)) => { + let size = t.size_vec2(); + if size.x > 0.0 && !ratios.0.contains_key(&item.token) { + ratios.0.insert(item.token.clone(), size.y / size.x); + } + egui::Image::new(&t).uv(cover_uv_for(size, rect.size())).corner_radius(theme::RADIUS).paint_at(ui, rect); + }, + Some(Lookup::Loading) => shimmer(ui, rect), + _ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), kind_icon(item.kind), theme::FAINT), + } + if item.kind == MediaKind::Video && !(view.autoplay && item.animated.is_some()) { + let badge = Rect::from_center_size(rect.center(), vec2(40.0, 40.0)); + ui.painter().circle_filled(badge.center(), 20.0, Color32::from_black_alpha(110)); + icons::paint(ui.painter(), badge.shrink(12.0).translate(vec2(1.5, 0.0)), Icon::Play, theme::fade(Color32::WHITE, 0.9)); + } +} + +/// An audio card's face (`GalleryCard`'s audio branch, a square tile): the title, a music note +/// and the waveform player along the bottom. +fn audio_face(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, audio: &mut AudioPlayer) { + let mut mesh = Mesh::default(); + let (top, bottom) = (theme::fade(theme::ACCENT, 0.14), theme::fade(theme::CONTROLS, 0.6)); + mesh.colored_vertex(rect.left_top(), top); + mesh.colored_vertex(rect.right_top(), top); + mesh.colored_vertex(rect.right_bottom(), bottom); + mesh.colored_vertex(rect.left_bottom(), bottom); + mesh.add_triangle(0, 1, 2); + mesh.add_triangle(0, 2, 3); + ui.painter().add(mesh); + // Clear of the selection checkbox when selecting. + let inset = if view.selecting { 36.0 } else { 12.0 }; + let title = item.prompt_token.as_deref().and_then(|t| view.prompts.get(t)).cloned().unwrap_or_else(|| fallback_title(item)); + let galley = clamp_lines(ui, &title, 13.0, theme::fade(Color32::WHITE, 0.85), rect.width() - inset - 12.0, 2); + ui.painter().galley(pos2(rect.left() + inset, rect.top() + 12.0), galley, Color32::WHITE); + waveform::music_tile(ui, Rect::from_center_size(rect.center() - vec2(0.0, 12.0), vec2(48.0, 48.0)), true); + let bar = Rect::from_min_max(pos2(rect.left() + 12.0, rect.bottom() - 56.0), pos2(rect.right() - 12.0, rect.bottom() - 12.0)); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar)); + waveform::player(&mut child, view.id.with(("player", &item.token)), audio, &item.full_url, item.duration_secs, false); +} + +/// The hover actions for an item: copy prompt (list rows, when the text is known), recreate, +/// make video, share and download. Audio only shares and downloads (`Audio v1`). +fn item_actions<'a>(view: &FeedView<'_>, item: &FeedItem, prompt: Option<&String>) -> Vec<(Icon, &'a str, FeedAction)> { + let mut buttons = Vec::new(); + if let Some(text) = prompt { + buttons.push((Icon::Copy, "Copy prompt", FeedAction::CopyPrompt(text.clone()))); + } + if item.prompt_token.is_some() && item.kind != MediaKind::Audio { + buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone()))); + } + if view.make_video && item.kind == MediaKind::Image { + buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone()))); + } + buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone()))); + buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone()))); + buttons +} + +/// What an item is called without its prompt: its title, else "Audio Generation" and the like. +fn fallback_title(item: &FeedItem) -> String { + item.title.clone().unwrap_or_else(|| format!("{} Generation", item.kind.label())) +} + +fn kind_icon(kind: MediaKind) -> Icon { + match kind { + MediaKind::Image => Icon::Image, + MediaKind::Video => Icon::Video, + MediaKind::Audio => Icon::Music, + } +} + fn pending_row(ui: &mut Ui, rect: Rect, job: &PendingJob, now: i64, names: &dyn ModelNames) { let thumb = Rect::from_min_size(rect.min + vec2(0.0, 12.0), vec2(LIST_THUMB, LIST_THUMB)); ui.painter().rect_filled(thumb, theme::RADIUS, Color32::from_white_alpha(8)); @@ -359,7 +427,7 @@ fn failed_row(ui: &mut Ui, id: Id, rect: Rect, job: &FailedJob, names: &dyn Mode } #[allow(clippy::too_many_arguments)] -fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, names: &dyn ModelNames, now: i64, actions: &mut Vec) { +fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, audio: &mut AudioPlayer, names: &dyn ModelNames, now: i64, actions: &mut Vec) { let resp = ui.interact(rect, view.id.with(("row", &item.token)), Sense::click()); let hovered = resp.hovered(); if hovered { @@ -367,7 +435,10 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache } let thumb = Rect::from_min_size(rect.min + vec2(0.0, 12.0), vec2(LIST_THUMB, LIST_THUMB)); ui.painter().rect_filled(thumb, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4)); - if let Some(Lookup::Ready(t)) = item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) { + let is_audio = item.kind == MediaKind::Audio; + if is_audio { + waveform::music_tile(ui, thumb, false); + } else if let Some(Lookup::Ready(t)) = item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) { egui::Image::new(&t).uv(crate::prompt_box::deck::cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, thumb); } if view.selected.contains(&item.token) { @@ -378,7 +449,8 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache let prompt = item.prompt_token.as_deref().and_then(|t| view.prompts.get(t)); match prompt { Some(text) => { - let g = clamp_lines(ui, text, 14.0, theme::INK, w, 3); + // Audio keeps room for its player under the text. + let g = clamp_lines(ui, text, 14.0, theme::INK, w, if is_audio { 2 } else { 3 }); ui.painter().galley(pos2(x, thumb.top()), g, theme::INK); }, None if item.prompt_token.is_some() => { @@ -387,9 +459,13 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache } }, None => { - ui.painter().text(pos2(x, thumb.top() + 8.0), Align2::LEFT_CENTER, format!("{} Generation", item.kind.label()), FontId::new(14.0, FontFamily::Proportional), theme::MUTED); + ui.painter().text(pos2(x, thumb.top() + 8.0), Align2::LEFT_CENTER, fallback_title(item), FontId::new(14.0, FontFamily::Proportional), theme::MUTED); }, } + if is_audio { + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(Rect::from_min_size(pos2(x, thumb.top() + 44.0), vec2(w.min(LIST_PLAYER_MAX), 32.0)))); + waveform::player(&mut child, view.id.with(("row-player", &item.token)), audio, &item.full_url, item.duration_secs, true); + } let meta_y = thumb.bottom() - 10.0; let mut mx = x; if let Some(model) = item.model_id.as_deref() { @@ -400,18 +476,7 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache ui.painter().text(pos2(rect.right(), thumb.top() + 8.0), Align2::RIGHT_CENTER, time_ago(item.created_at, now), FontId::new(11.0, FontFamily::Proportional), theme::FAINT); let mut action_clicked = false; if hovered && !view.selecting { - let mut buttons: Vec<(Icon, &str, FeedAction)> = Vec::new(); - if let Some(text) = prompt { - buttons.push((Icon::Copy, "Copy prompt", FeedAction::CopyPrompt(text.clone()))); - } - if item.prompt_token.is_some() { - buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone()))); - } - if view.make_video && item.kind == MediaKind::Image { - buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone()))); - } - buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone()))); - buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone()))); + let buttons = item_actions(view, item, prompt); let bar = Rect::from_min_size(pos2(rect.right() - 28.0 * buttons.len() as f32, thumb.bottom() - 28.0), vec2(28.0 * buttons.len() as f32, 28.0)); let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar).layout(egui::Layout::left_to_right(egui::Align::Center))); child.spacing_mut().item_spacing.x = 0.0; @@ -464,9 +529,15 @@ fn model_chip(ui: &Ui, left_center: egui::Pos2, model_id: &str, names: &dyn Mode /// Darkens the bottom of `rect` (`bg-gradient-to-t from-black/70`), returning the faded band. fn bottom_gradient(ui: &Ui, rect: Rect, height: f32, alpha: u8) -> Rect { - let band = Rect::from_min_max(pos2(rect.left(), (rect.bottom() - height).max(rect.top())), rect.max); + edge_fade(ui, rect, height, alpha, false) +} + +/// Darkens `rect` toward its top or bottom edge over `height`, returning the faded band. +fn edge_fade(ui: &Ui, rect: Rect, height: f32, alpha: u8, top_edge: bool) -> Rect { + let band = if top_edge { Rect::from_min_max(rect.min, pos2(rect.right(), (rect.top() + height).min(rect.bottom()))) } else { Rect::from_min_max(pos2(rect.left(), (rect.bottom() - height).max(rect.top())), rect.max) }; let mut mesh = Mesh::default(); - let (top, bottom) = (Color32::TRANSPARENT, Color32::from_black_alpha(alpha)); + let (clear, dark) = (Color32::TRANSPARENT, Color32::from_black_alpha(alpha)); + let (top, bottom) = if top_edge { (dark, clear) } else { (clear, dark) }; mesh.colored_vertex(band.left_top(), top); mesh.colored_vertex(band.right_top(), top); mesh.colored_vertex(band.right_bottom(), bottom); diff --git a/artcraft-native/src/feed/store.rs b/artcraft-native/src/feed/store.rs index 43f281101c..64b3c67d9c 100644 --- a/artcraft-native/src/feed/store.rs +++ b/artcraft-native/src/feed/store.rs @@ -6,12 +6,14 @@ use std::collections::{HashMap, HashSet}; use crate::backend::wire::{Job, JobPhase, MediaFile, unix_secs}; use crate::backend::EnqueueMeta; use crate::feed::types::{FailedJob, FeedItem, MediaKind, PendingJob}; -use crate::models::Catalog; +use crate::models::{self, Catalog}; /// Failures older than this are history, not news. const FAILURE_WINDOW_SECS: i64 = 24 * 60 * 60; /// Thumbnails are requested at this width (`getMediaThumbnail(@512)`). pub const THUMBNAIL_WIDTH: u32 = 512; +/// Animated previews are requested smaller: every frame becomes a texture. +const ANIMATED_WIDTH: u32 = 320; /// Something the store wants the app to do or say after an update. #[derive(Debug, PartialEq)] @@ -54,7 +56,7 @@ impl FeedStore { /// Shows the jobs a generate call just started, before the next poll catches up. pub fn add_enqueued(&mut self, kind: MediaKind, job_tokens: &[String], meta: &EnqueueMeta, catalog: &Catalog) { let now = chrono::Utc::now().timestamp(); - let expected = catalog.find(&meta.model_id).map_or(30.0, |m| m.expected_secs); + let expected = catalog.find(&meta.model_id).map_or_else(|| models::expected_secs(&meta.model_id, kind), |m| m.expected_secs); for token in job_tokens { self.meta.insert(token.clone(), meta.clone()); self.seen_running.insert(token.clone()); @@ -79,7 +81,7 @@ impl FeedStore { match job.status.phase() { JobPhase::Running => { self.seen_running.insert(job.job_token.clone()); - let expected = model_id.as_deref().and_then(|id| catalog.find(id)).map_or(if kind == MediaKind::Video { 900.0 } else { 30.0 }, |m| m.expected_secs); + let expected = model_id.as_deref().and_then(|id| catalog.find(id)).map_or_else(|| models::expected_secs(model_id.as_deref().unwrap_or_default(), kind), |m| m.expected_secs); pending.push(PendingJob { job_token: job.job_token.clone(), kind, prompt, model_id, created_at: unix_secs(&job.created_at), server_progress: Some(job.status.progress_percentage), expected_secs: expected, batch_count: meta.map_or(1, |m| m.batch_count) }); }, JobPhase::Failed => { @@ -103,7 +105,8 @@ impl FeedStore { }; // The finished card takes the pending card's place: date it by the job. let created_at = unix_secs(&job.created_at); - self.push_item(FeedItem { token: result.entity_token.clone(), kind, thumbnail: result.media_links.thumbnail(THUMBNAIL_WIDTH), full_url: result.media_links.cdn_url.clone(), created_at, model_id: model_id.clone(), prompt_token: job.request.maybe_prompt_token.clone(), batch_token: result.maybe_batch_token.clone(), duration_secs: None }); + let thumbnail = result.media_links.thumbnail(THUMBNAIL_WIDTH).filter(|_| kind != MediaKind::Audio); + self.push_item(FeedItem { token: result.entity_token.clone(), kind, title: None, thumbnail, full_url: result.media_links.cdn_url.clone(), created_at, model_id: model_id.clone(), prompt_token: job.request.maybe_prompt_token.clone(), batch_token: result.maybe_batch_token.clone(), duration_secs: None, animated: result.media_links.animated_preview(ANIMATED_WIDTH) }); if let Some(batch) = &result.maybe_batch_token { notices.push(FeedNotice::LoadBatch { job_token: job.job_token.clone(), batch_token: batch.clone() }); } @@ -207,19 +210,23 @@ pub fn feed_item(kind: MediaKind, file: &MediaFile) -> FeedItem { kind: match file.media_class.as_deref() { Some("video") => MediaKind::Video, Some("image") => MediaKind::Image, + Some("audio") => MediaKind::Audio, _ => kind, }, - thumbnail: file.media_links.thumbnail(THUMBNAIL_WIDTH), + title: file.maybe_title.clone().filter(|t| !t.is_empty()), + thumbnail: file.thumbnail(THUMBNAIL_WIDTH), full_url: file.media_links.cdn_url.clone(), created_at: unix_secs(&file.created_at), model_id: file.maybe_model_type.clone().or_else(|| file.maybe_origin_model_type.clone()), prompt_token: file.maybe_prompt_token.clone(), batch_token: file.maybe_batch_token.clone(), duration_secs: file.maybe_duration_millis.map(|ms| ms as f32 / 1000.0), + animated: file.media_links.animated_preview(ANIMATED_WIDTH), } } -/// Whether a job made an image or a video: by its model, else by its category. +/// Whether a job made an image, a video or audio: by its model, else by its category +/// (`useGenerationJobs` checks video before image before audio). fn job_kind(job: &Job, catalog: &Catalog) -> Option { if let Some(model) = job.request.maybe_model_type.as_deref() { if catalog.image.iter().any(|m| m.id == model) { @@ -228,12 +235,17 @@ fn job_kind(job: &Job, catalog: &Catalog) -> Option { if catalog.video.iter().any(|m| m.id == model) { return Some(MediaKind::Video); } + if catalog.audio.iter().any(|m| m.id == model) { + return Some(MediaKind::Audio); + } } let category = job.request.inference_category.as_str(); if category.contains("video") { Some(MediaKind::Video) } else if category.contains("image") { Some(MediaKind::Image) + } else if category.contains("audio") { + Some(MediaKind::Audio) } else { None } diff --git a/artcraft-native/src/feed/types.rs b/artcraft-native/src/feed/types.rs index ee3ba9f209..921f1793d1 100644 --- a/artcraft-native/src/feed/types.rs +++ b/artcraft-native/src/feed/types.rs @@ -4,6 +4,7 @@ pub enum MediaKind { Image, Video, + Audio, } impl MediaKind { @@ -11,6 +12,7 @@ impl MediaKind { match self { MediaKind::Image => "Image", MediaKind::Video => "Video", + MediaKind::Audio => "Audio", } } @@ -18,17 +20,20 @@ impl MediaKind { match self { MediaKind::Image => "images", MediaKind::Video => "videos", + MediaKind::Audio => "audio clips", } } } -/// A finished image or video in the user's library. +/// A finished image, video or audio clip in the user's library. #[derive(Clone, Debug)] pub struct FeedItem { /// The media file token. pub token: String, pub kind: MediaKind, - /// A thumbnail URL (512 px wide where the CDN can resize). + /// The file's title (`maybe_title`), when it has one. + pub title: Option, + /// A thumbnail URL (512 px wide where the CDN can resize); audio has none. pub thumbnail: Option, /// The original file. pub full_url: String, @@ -39,6 +44,8 @@ pub struct FeedItem { pub prompt_token: Option, pub batch_token: Option, pub duration_secs: Option, + /// A video's animated preview (shown when previews play). + pub animated: Option, } /// A generation the server is still working on. diff --git a/artcraft-native/src/models.rs b/artcraft-native/src/models.rs index 84fd713a05..2586d5da44 100644 --- a/artcraft-native/src/models.rs +++ b/artcraft-native/src/models.rs @@ -8,10 +8,15 @@ use crate::feed::types::MediaKind; /// Default models (`defaultModelForPage.ts`). pub const DEFAULT_IMAGE_MODEL: &str = "nano_banana_pro"; pub const DEFAULT_VIDEO_MODEL: &str = "seedance_2p0"; +pub const DEFAULT_AUDIO_MODEL: &str = "suno_music"; /// The fallback prompt limit (`maxPromptLength ?? 1000`). pub const DEFAULT_PROMPT_MAX: usize = 1000; /// The model pickers group by family once they list this many models. const GROUP_THRESHOLD: usize = 8; +/// Audio models that work from exactly one audio track (`AUDIO_MODELS_REQUIRING_AUDIO_REF`). +const AUDIO_MODELS_REQUIRING_AUDIO_REF: [&str; 2] = ["suno_remix", "suno_sample"]; +/// The musical keys (`SoundsSettingsPopover`; there are no E keys): value, label, short label. +pub const MUSICAL_KEYS: [(&str, &str, &str); 13] = [("auto", "Auto", "Auto"), ("c_major", "C Major", "C"), ("c_minor", "C Minor", "Cm"), ("d_major", "D Major", "D"), ("d_minor", "D Minor", "Dm"), ("f_major", "F Major", "F"), ("f_minor", "F Minor", "Fm"), ("g_major", "G Major", "G"), ("g_minor", "G Minor", "Gm"), ("a_major", "A Major", "A"), ("a_minor", "A Minor", "Am"), ("b_major", "B Major", "B"), ("b_minor", "B Minor", "Bm")]; /// One generation model and what it accepts. #[derive(Clone, Debug, Default)] @@ -45,6 +50,8 @@ pub struct ModelInfo { pub video_refs_max_secs: Option, pub audio_refs_max: usize, pub audio_refs_max_secs: Option, + /// `@Character` mentions (0 = not supported). + pub character_refs_max: usize, pub duration_options: Vec, pub duration_min: Option, pub duration_max: Option, @@ -54,10 +61,42 @@ pub struct ModelInfo { pub bitrate_default: Option, pub output_formats: Vec, pub output_format_default: Option, + /// Audio only. + pub audio: AudioCaps, /// How long a generation usually takes (drives the estimated progress bar). pub expected_secs: f32, } +/// What an audio model takes besides the prompt and references. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AudioCaps { + pub style_prompt: bool, + pub instrumental: bool, + pub keep_lyrics: bool, + pub loopable: bool, + pub bpm: bool, + pub musical_key: bool, + pub sample_rates: Vec, + pub sample_rate_default: Option, + pub speed: bool, + pub volume: bool, + pub pitch: bool, + /// Remixes and samples work from exactly one audio track. + pub requires_audio_ref: bool, +} + +impl AudioCaps { + /// Whether the Tuning popover shows. + pub fn has_tuning(&self) -> bool { + !self.sample_rates.is_empty() || self.speed || self.volume || self.pitch + } + + /// `picked` if the model offers it, else its default, else its first rate. + pub fn sample_rate(&self, picked: Option) -> Option { + picked.filter(|hz| self.sample_rates.contains(hz)).or(self.sample_rate_default).or_else(|| self.sample_rates.first().copied()) + } +} + impl ModelInfo { /// Batch sizes the count picker offers (`predefined` or `1..=max`). pub fn batch_options(&self) -> Vec { @@ -162,6 +201,7 @@ pub fn video_model(d: &artcraft_client::endpoints::omni_gen::models::video::omni m.video_refs_max_secs = d.video_references_max_total_duration_seconds; m.audio_refs_max = max(d.audio_references_supported, d.audio_references_max); m.audio_refs_max_secs = d.audio_references_max_total_duration_seconds; + m.character_refs_max = max(d.character_references_supported, d.character_references_max); m.aspect_ratios = strings(&d.aspect_ratio_options); m.aspect_default = d.aspect_ratio_default.clone().map(String::from); m.resolutions = strings(&d.resolution_options); @@ -182,6 +222,22 @@ pub fn video_model(d: &artcraft_client::endpoints::omni_gen::models::video::omni m } +/// Builds a model from the server's audio listing. +pub fn audio_model(d: &crate::backend::wire::AudioModel) -> ModelInfo { + let mut m = base(&d.model, MediaKind::Audio, d.full_name.as_deref(), d.model_creator.clone()); + if let Some(short) = d.extra_info_short.as_deref().filter(|s| !s.is_empty()) { + m.description = short.to_owned(); + } + m.disabled = d.is_disabled; + m.prompt_supported = d.text_prompt_supported.unwrap_or(true); + m.prompt_max = None; + let max = |supported: bool, max: Option| if supported { max.map_or(1, usize::from) } else { 0 }; + m.audio_refs_max = max(d.audio_references_supported, d.audio_references_max); + m.image_refs_max = max(d.image_references_supported, d.image_references_max); + m.audio = AudioCaps { style_prompt: d.style_prompt_supported, instrumental: d.instrumental_toggle_supported, keep_lyrics: d.keep_lyrics_supported, loopable: d.loopable_toggle_supported, bpm: d.bpm_supported, musical_key: d.musical_key_supported, sample_rates: d.sample_rate_hz_options.clone(), sample_rate_default: d.sample_rate_hz_default, speed: d.speed_supported, volume: d.volume_supported, pitch: d.pitch_supported, requires_audio_ref: AUDIO_MODELS_REQUIRING_AUDIO_REF.contains(&d.model.as_str()) }; + m +} + fn strings>(options: &Option>) -> Vec { options.as_deref().unwrap_or_default().iter().cloned().map(Into::into).collect() } @@ -209,7 +265,7 @@ fn base(id: &str, kind: MediaKind, full_name: Option<&str>, creator: Option bool { - const PREFIXES: [&str; 11] = ["flux_", "nano_banana", "gpt_image", "seedream", "seedance", "kling", "veo_", "sora", "happy_horse", "grok_imagine", "qwen"]; + const PREFIXES: [&str; 13] = ["flux_", "nano_banana", "gpt_image", "seedream", "seedance", "kling", "veo_", "sora", "happy_horse", "grok_imagine", "qwen", "suno", "seed_audio"]; PREFIXES.iter().any(|p| id.starts_with(p)) } @@ -289,12 +345,17 @@ fn description(id: &str) -> &'static str { "grok_video" | "grok_imagine_video" => "Fastest video model", "sora_2" => "Smart video model", "veo_3" => "Slow, high-quality model", + "suno_music" => "Full songs from a text prompt", + "suno_remix" => "Remix an existing track", + "suno_sounds" => "Sound effects with beat control", + "suno_sample" => "Build a song from a sample", + "seed_audio_1p0" => "Sound generation with fine tuning", _ => "", } } -/// Typical generation time (the static list's `progressBarTime`). -fn expected_secs(id: &str, kind: MediaKind) -> f32 { +/// Typical generation time (the static list's `progressBarTime`, else the feed's per-kind default). +pub fn expected_secs(id: &str, kind: MediaKind) -> f32 { match id { "grok_image" | "grok_imagine_image" | "flux_pro_1p1" | "flux_1_dev" | "flux_1_schnell" => 10.0, "grok_video" | "grok_imagine_video" => 50.0, @@ -304,19 +365,20 @@ fn expected_secs(id: &str, kind: MediaKind) -> f32 { "gpt_image_1" | "gpt_image_1p5" | "gpt_image_2p5_flare" | "seedream_4" | "seedream_4p5" | "seedream_5_lite" => 60.0, "gpt_image_2" | "gpt_image_2p5_sunburst" => 120.0, _ if kind == MediaKind::Video => 900.0, + _ if kind == MediaKind::Audio => 120.0, _ => 30.0, } } /// The maker, by id prefix, when the server doesn't say. pub fn creator_for(id: &str) -> &'static str { - const PREFIXES: [(&str, &str); 16] = [("flux", "black_forest_labs"), ("nano_banana", "google"), ("gemini", "google"), ("veo", "google"), ("imagen", "google"), ("gpt_image", "openai"), ("sora", "openai"), ("seedream", "bytedance"), ("seedance", "bytedance"), ("seededit", "bytedance"), ("kling", "kling"), ("grok", "grok"), ("midjourney", "midjourney"), ("qwen", "alibaba"), ("happy_horse", "alibaba"), ("minimax", "minimax")]; + const PREFIXES: [(&str, &str); 18] = [("suno", "suno"), ("seed_audio", "bytedance"), ("flux", "black_forest_labs"), ("nano_banana", "google"), ("gemini", "google"), ("veo", "google"), ("imagen", "google"), ("gpt_image", "openai"), ("sora", "openai"), ("seedream", "bytedance"), ("seedance", "bytedance"), ("seededit", "bytedance"), ("kling", "kling"), ("grok", "grok"), ("midjourney", "midjourney"), ("qwen", "alibaba"), ("happy_horse", "alibaba"), ("minimax", "minimax")]; PREFIXES.iter().find(|(p, _)| id.starts_with(p)).map_or("artcraft", |(_, c)| c) } /// The picker family (`getModelFamilyName`). fn family(id: &str) -> &'static str { - const FAMILIES: [(&str, &str); 22] = [("seedance", "Seedance"), ("kling", "Kling"), ("veo", "Veo"), ("sora", "Sora"), ("happy_horse", "Happy Horse"), ("wan", "Wan"), ("vidu", "Vidu"), ("preview", "Preview"), ("nano_banana", "Nano Banana"), ("gemini_25_flash", "Nano Banana"), ("seedream", "Seedream"), ("seededit", "SeedEdit"), ("gpt_image", "GPT Image"), ("flux", "Flux"), ("midjourney", "Midjourney"), ("grok", "Grok"), ("qwen", "Qwen"), ("recraft", "Recraft"), ("imagen", "Imagen"), ("minimax", "MiniMax"), ("marble", "World Labs"), ("suno", "Suno")]; + const FAMILIES: [(&str, &str); 23] = [("seed_audio", "Seed Audio"), ("seedance", "Seedance"), ("kling", "Kling"), ("veo", "Veo"), ("sora", "Sora"), ("happy_horse", "Happy Horse"), ("wan", "Wan"), ("vidu", "Vidu"), ("preview", "Preview"), ("nano_banana", "Nano Banana"), ("gemini_25_flash", "Nano Banana"), ("seedream", "Seedream"), ("seededit", "SeedEdit"), ("gpt_image", "GPT Image"), ("flux", "Flux"), ("midjourney", "Midjourney"), ("grok", "Grok"), ("qwen", "Qwen"), ("recraft", "Recraft"), ("imagen", "Imagen"), ("minimax", "MiniMax"), ("marble", "World Labs"), ("suno", "Suno")]; FAMILIES.iter().find(|(p, _)| id.starts_with(p)).map_or("Other", |(_, f)| f) } @@ -386,13 +448,16 @@ fn video_page_models(models: &[ModelInfo]) -> Vec { list } -/// Both catalogs, for looking up names in the feed, and each create page's picker list. +/// Every catalog, for looking up names in the feed, and each create page's picker list. #[derive(Default)] pub struct Catalog { pub image: Vec, pub video: Vec, + pub audio: Vec, pub image_page: Vec, pub video_page: Vec, + /// The audio page's list: enabled, in the server's order. + pub audio_page: Vec, } impl Catalog { @@ -406,8 +471,13 @@ impl Catalog { self.video = models; } + pub fn set_audio(&mut self, models: Vec) { + self.audio_page = models.iter().filter(|m| !m.disabled).cloned().collect(); + self.audio = models; + } + pub fn find(&self, id: &str) -> Option<&ModelInfo> { - self.image.iter().chain(&self.video).find(|m| m.id == id) + self.image.iter().chain(&self.video).chain(&self.audio).find(|m| m.id == id) } } @@ -502,6 +572,15 @@ pub fn quality_label(value: &str) -> String { .to_owned() } +/// Sample rate labels (`formatSampleRateHz`): 44100 → "44.1 kHz", 24000 → "24 kHz". +pub fn sample_rate_label(hz: u32) -> String { + if hz.is_multiple_of(1000) { + format!("{} kHz", hz / 1000) + } else { + format!("{:.1} kHz", hz as f32 / 1000.0) + } +} + /// Bitrate labels. pub fn bitrate_label(value: &str) -> String { match value { @@ -530,6 +609,8 @@ mod tests { assert_eq!(display_name("kling_3p0_pro"), "Kling 3.0 Pro"); assert_eq!(display_name("nano_banana"), "Nano Banana"); assert_eq!(display_name("seedance_2p0_bpu"), "Seedance 2.0 Plus Ultra"); + assert_eq!(display_name("seed_audio_1p0"), "Seed Audio 1.0"); + assert_eq!(display_name("suno_music"), "Suno Music"); } #[test] @@ -544,6 +625,8 @@ mod tests { #[test] fn falls_back_to_creator_by_prefix() { + assert_eq!(creator_for("seed_audio_1p0"), "bytedance"); + assert_eq!(creator_for("suno_remix"), "suno"); assert_eq!(creator_for("seedream_4p5"), "bytedance"); assert_eq!(creator_for("veo_3"), "google"); assert_eq!(creator_for("something_new"), "artcraft"); @@ -573,6 +656,21 @@ mod tests { assert_eq!(ModelInfo::resolve(None, &[], None), None); } + #[test] + fn audio_listing_maps_capabilities() { + let d: crate::backend::wire::AudioModel = serde_json::from_value(serde_json::json!({ "model": "seed_audio_1p0", "model_creator": "artcraft", "audio_references_supported": true, "audio_references_max": 3, "image_references_supported": true, "sample_rate_hz_options": [8000, 24000, 44100], "sample_rate_hz_default": 24000, "speed_supported": true })).unwrap(); + let m = audio_model(&d); + assert_eq!((m.name.as_str(), m.creator.as_str(), m.description.as_str()), ("Seed Audio 1.0", "bytedance", "Sound generation with fine tuning")); + assert_eq!((m.audio_refs_max, m.image_refs_max, m.prompt_max), (3, 1, None)); + assert!(m.audio.has_tuning() && !m.audio.requires_audio_ref && !m.audio.style_prompt); + assert_eq!(m.audio.sample_rate(Some(44100)), Some(44100)); + assert_eq!(m.audio.sample_rate(Some(11025)), Some(24000), "unsupported picks fall back to the default"); + let remix = audio_model(&serde_json::from_value(serde_json::json!({ "model": "suno_remix", "audio_references_supported": true })).unwrap()); + assert!(remix.audio.requires_audio_ref && !remix.audio.has_tuning()); + assert_eq!((remix.audio_refs_max, remix.audio.sample_rate(None)), (1, None)); + assert_eq!((sample_rate_label(44100).as_str(), sample_rate_label(24000).as_str()), ("44.1 kHz", "24 kHz")); + } + #[test] fn batch_counts_snap_to_valid_values() { let mut m = model("grok_imagine_image"); diff --git a/artcraft-native/src/overlays/characters.rs b/artcraft-native/src/overlays/characters.rs new file mode 100644 index 0000000000..f02325686a --- /dev/null +++ b/artcraft-native/src/overlays/characters.rs @@ -0,0 +1,402 @@ +//! The characters dialog (`CharactersModal`): pick a saved character to mention it in the prompt, +//! make a new one from a reference image, rename or re-describe one, or delete it. + +use std::path::PathBuf; +use std::sync::Arc; + +use egui::{Align2, Color32, FontId, Id, Rect, Sense, Stroke, Ui, pos2, vec2}; + +use crate::backend::UploadedMedia; +use crate::backend::media_cache::{Lookup, MediaCache}; +use crate::backend::wire::Character; +use crate::prompt_box::deck::cover_uv; +use crate::prompt_box::refs::{RefKind, RefMedia, RefStatus}; +use crate::theme; +use crate::ui::icons::{self, Icon}; +use crate::ui::widgets; + +const COLUMNS: usize = 4; +const GAP: f32 = 12.0; + +#[derive(Clone, PartialEq)] +enum View { + List, + Edit(String), + Create, +} + +pub struct CharactersModal { + view: View, + confirm_delete: Option<(String, String)>, + name: String, + description: String, + image: Option, +} + +/// Something only the app can do for the dialog. +#[derive(Debug)] +pub enum CharactersAction { + Close, + /// Mention this character in the prompt. + Select { + token: String, + name: String, + }, + Upload { + ref_id: u64, + path: PathBuf, + }, + Create { + image_token: String, + name: String, + description: Option, + }, + Edit { + token: String, + name: String, + description: String, + }, + Delete { + token: String, + name: String, + }, + Preview(String), + Toast(String), +} + +impl CharactersModal { + pub fn new() -> Self { + Self { view: View::List, confirm_delete: None, name: String::new(), description: String::new(), image: None } + } + + /// The reference image being uploaded for a new character, if `ref_id` is it. + pub fn apply_upload(&mut self, ref_id: u64, result: Result) -> Option> { + let image = self.image.as_mut().filter(|i| i.id == ref_id)?; + Some(match result { + Ok(media) => { + image.status = RefStatus::Ready; + image.token = Some(media.token); + image.full_url = media.full_url; + Ok(()) + }, + Err(err) => { + self.image = None; + Err(err) + }, + }) + } + + pub fn show(&mut self, ctx: &egui::Context, characters: &[Character], pending: &[String], cache: &mut MediaCache) -> Vec { + let mut actions = Vec::new(); + let width = (ctx.content_rect().width() - 80.0).min(760.0); + let close = widgets::modal(ctx, Id::new("characters"), width, |ui| match self.view.clone() { + View::List => self.list(ui, characters, pending, cache, &mut actions), + View::Edit(token) => match characters.iter().find(|c| c.token == token) { + Some(c) => self.edit(ui, c, cache, &mut actions), + None => self.view = View::List, + }, + View::Create => self.create(ui, cache, &mut actions), + }); + if close && self.confirm_delete.is_none() { + actions.push(CharactersAction::Close); + } + if let Some((token, name)) = self.confirm_delete.clone() { + let mut choice = None; + let dismissed = widgets::modal(ctx, Id::new("character-delete"), 420.0, |ui| { + ui.label(widgets::heading("Delete character?", 20.0)); + ui.add_space(8.0); + ui.label(egui::RichText::new(format!("\u{201c}{name}\u{201d} will be removed from your characters.")).color(theme::MUTED)); + ui.add_space(16.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if widgets::button(ui, None, "Delete", widgets::Kind::Danger, 0.0, theme::CONTROL_H).clicked() { + choice = Some(true); + } + if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() { + choice = Some(false); + } + }); + }); + if dismissed || choice.is_some() { + self.confirm_delete = None; + } + if choice == Some(true) { + actions.push(CharactersAction::Delete { token, name }); + } + } + actions + } + + fn header(&mut self, ui: &mut Ui, title: &str, back: bool, actions: &mut Vec) { + ui.horizontal(|ui| { + if back && widgets::ghost(ui, Icon::ChevronLeft, None, "Back", false).clicked() { + self.view = View::List; + } + ui.label(widgets::heading(title, 22.0)); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if widgets::ghost(ui, Icon::X, None, "Close", false).clicked() { + actions.push(CharactersAction::Close); + } + }); + }); + ui.add_space(12.0); + } + + fn list(&mut self, ui: &mut Ui, characters: &[Character], pending: &[String], cache: &mut MediaCache, actions: &mut Vec) { + self.header(ui, "Characters", false, actions); + let tile = (ui.available_width() - GAP * (COLUMNS - 1) as f32) / COLUMNS as f32; + let max_h = ui.ctx().content_rect().height() * 0.6; + egui::ScrollArea::vertical().max_height(max_h).auto_shrink([false, true]).show(ui, |ui| { + let total = 1 + pending.len() + characters.len(); + let rows = total.div_ceil(COLUMNS); + let origin = ui.cursor().min; + ui.allocate_exact_size(vec2(ui.available_width(), rows as f32 * (tile + 30.0 + GAP)), Sense::hover()); + let cell = |i: usize| Rect::from_min_size(origin + vec2((i % COLUMNS) as f32 * (tile + GAP), (i / COLUMNS) as f32 * (tile + 30.0 + GAP)), vec2(tile, tile)); + // "Create New" first. + let create = cell(0); + let resp = ui.interact(create, Id::new("character-create"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand); + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, "Create New")); + ui.painter().rect_filled(create, theme::RADIUS, theme::fade(theme::CONTROLS, if resp.hovered() { 0.7 } else { 0.4 })); + icons::dashed_rect(ui.painter(), create.shrink(1.0), Stroke::new(1.5, theme::LINE_DASHED), 5.0, 4.0); + icons::paint(ui.painter(), Rect::from_center_size(create.center() - vec2(0.0, 10.0), vec2(22.0, 22.0)), Icon::Plus, theme::INK); + ui.painter().text(create.center() + vec2(0.0, 16.0), Align2::CENTER_CENTER, "Create New", FontId::new(13.0, theme::medium()), theme::INK); + if resp.clicked() { + self.name.clear(); + self.description.clear(); + self.image = None; + self.view = View::Create; + } + for (i, name) in pending.iter().enumerate() { + let r = cell(1 + i); + ui.painter().rect_filled(r, theme::RADIUS, theme::WASH); + icons::paint_spinner(ui, Rect::from_center_size(r.center(), vec2(24.0, 24.0)), theme::MUTED); + caption(ui, r, name, "Creating..."); + } + for (i, c) in characters.iter().enumerate() { + let r = cell(1 + pending.len() + i); + let resp = ui.interact(r, Id::new(("character", &c.token)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand); + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, &c.name)); + thumbnail(ui, r, c.avatar_url().as_deref(), cache); + if resp.hovered() { + ui.painter().rect_stroke(r, theme::RADIUS, Stroke::new(2.0, theme::fade(theme::ACCENT_400, 0.7)), egui::StrokeKind::Inside); + } + caption(ui, r, &c.name, ""); + let mut clicked_tool = false; + if ui.rect_contains_pointer(r) { + let tools = Rect::from_min_size(pos2(r.right() - 64.0, r.top() + 6.0), vec2(58.0, 28.0)); + ui.painter().rect_filled(tools, theme::RADIUS, Color32::from_black_alpha(160)); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(tools.shrink(1.0)).layout(egui::Layout::left_to_right(egui::Align::Center))); + child.spacing_mut().item_spacing.x = 0.0; + if widgets::overlay_icon(&mut child, Icon::Pencil, "Edit character").clicked() { + self.name = c.name.clone(); + self.description = c.maybe_description.clone().unwrap_or_default(); + self.view = View::Edit(c.token.clone()); + clicked_tool = true; + } + if widgets::overlay_icon(&mut child, Icon::Trash, "Delete character").clicked() { + self.confirm_delete = Some((c.token.clone(), c.name.clone())); + clicked_tool = true; + } + } + if resp.clicked() && !clicked_tool { + actions.push(CharactersAction::Select { token: c.token.clone(), name: c.name.clone() }); + } + } + }); + } + + fn edit(&mut self, ui: &mut Ui, c: &Character, cache: &mut MediaCache, actions: &mut Vec) { + self.header(ui, "Edit Character", true, actions); + ui.horizontal_top(|ui| { + ui.vertical(|ui| { + let (r, _) = ui.allocate_exact_size(vec2(240.0, 240.0), Sense::hover()); + thumbnail(ui, r, c.full_url().or_else(|| c.avatar_url()).as_deref(), cache); + if let Some(url) = c.full_url() { + if ui.add(egui::Button::new(egui::RichText::new("View full size").size(12.0).color(theme::ACCENT_INK)).frame(false)).clicked() { + actions.push(CharactersAction::Preview(url)); + } + } + }); + ui.add_space(16.0); + ui.vertical(|ui| { + fields(ui, &mut self.name, &mut self.description); + ui.add_space(16.0); + ui.horizontal(|ui| { + if widgets::button(ui, None, "Save", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() { + if self.name.trim().is_empty() { + actions.push(CharactersAction::Toast("Name cannot be empty".to_owned())); + } else { + actions.push(CharactersAction::Edit { token: c.token.clone(), name: self.name.trim().to_owned(), description: self.description.trim().to_owned() }); + self.view = View::List; + } + } + if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() { + self.view = View::List; + } + }); + }); + }); + } + + fn create(&mut self, ui: &mut Ui, cache: &mut MediaCache, actions: &mut Vec) { + self.header(ui, "New Character", true, actions); + ui.horizontal_top(|ui| { + ui.vertical(|ui| { + widgets::hud(ui, "Reference", theme::FAINT); + ui.add_space(6.0); + let (r, resp) = ui.allocate_exact_size(vec2(240.0, 240.0), Sense::click()); + match &self.image { + Some(img) => { + thumbnail(ui, r, img.preview.as_deref(), cache); + if img.status == RefStatus::Uploading { + ui.painter().rect_filled(r, theme::RADIUS, Color32::from_black_alpha(110)); + icons::paint_spinner(ui, Rect::from_center_size(r.center(), vec2(28.0, 28.0)), Color32::WHITE); + } + }, + None => { + ui.painter().rect_filled(r, theme::RADIUS, theme::fade(theme::CONTROLS, if resp.hovered() { 0.7 } else { 0.4 })); + icons::dashed_rect(ui.painter(), r.shrink(1.0), Stroke::new(1.5, theme::LINE_DASHED), 5.0, 4.0); + icons::paint(ui.painter(), Rect::from_center_size(r.center() - vec2(0.0, 12.0), vec2(24.0, 24.0)), Icon::Upload, theme::INK); + ui.painter().text(r.center() + vec2(0.0, 18.0), Align2::CENTER_CENTER, "Upload reference image", FontId::new(13.0, theme::medium()), theme::INK); + }, + } + if resp.on_hover_cursor(egui::CursorIcon::PointingHand).clicked() { + if let Some(path) = crate::pages::common::pick_files(&[RefKind::Image], false).into_iter().next() { + match std::fs::read(&path) { + Ok(bytes) => { + let mut img = RefMedia::uploading(RefKind::Image, None); + let key = format!("local://{}", img.id); + cache.insert_bytes(ui.ctx(), &key, Arc::from(bytes)); + img.preview = Some(key); + actions.push(CharactersAction::Upload { ref_id: img.id, path }); + self.image = Some(img); + }, + Err(err) => actions.push(CharactersAction::Toast(format!("Failed to upload {}: {err}", crate::pages::common::file_name(&path)))), + } + } + } + }); + ui.add_space(16.0); + ui.vertical(|ui| { + fields(ui, &mut self.name, &mut self.description); + ui.add_space(16.0); + let uploading = self.image.as_ref().is_some_and(|i| i.status == RefStatus::Uploading); + ui.horizontal(|ui| { + if ui.add_enabled_ui(!uploading, |ui| widgets::button(ui, Some(Icon::Sparkles), "Create", widgets::Kind::Primary, 0.0, theme::CONTROL_H)).inner.clicked() { + match (self.name.trim(), self.image.as_ref().and_then(|i| i.token.clone())) { + ("", _) => actions.push(CharactersAction::Toast("Please enter a character name".to_owned())), + (_, None) => actions.push(CharactersAction::Toast("Please upload a reference image".to_owned())), + (name, Some(image_token)) => { + let description = Some(self.description.trim().to_owned()).filter(|d| !d.is_empty()); + actions.push(CharactersAction::Create { image_token, name: name.to_owned(), description }); + self.view = View::List; + }, + } + } + if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() { + self.view = View::List; + } + }); + }); + }); + } +} + +/// The name and description inputs shared by create and edit. +fn fields(ui: &mut Ui, name: &mut String, description: &mut String) { + widgets::hud(ui, "Character name", theme::FAINT); + ui.add_space(4.0); + input_frame(ui, |ui| ui.add(egui::TextEdit::singleline(name).frame(egui::Frame::NONE).desired_width(f32::INFINITY).hint_text(egui::RichText::new("Character name").color(theme::FAINT)))); + ui.add_space(10.0); + widgets::hud(ui, "Description", theme::FAINT); + ui.add_space(4.0); + input_frame(ui, |ui| ui.add(egui::TextEdit::multiline(description).frame(egui::Frame::NONE).desired_width(f32::INFINITY).desired_rows(4).hint_text(egui::RichText::new("Description...").color(theme::FAINT)))); +} + +fn input_frame(ui: &mut Ui, add: impl FnOnce(&mut Ui) -> egui::Response) { + egui::Frame::new().fill(theme::SUNKEN).stroke(theme::hairline()).corner_radius(theme::RADIUS).inner_margin(egui::Margin::symmetric(10, 8)).show(ui, |ui| add(ui)); +} + +fn thumbnail(ui: &Ui, rect: Rect, url: Option<&str>, cache: &mut MediaCache) { + ui.painter().rect_filled(rect, theme::RADIUS, theme::WASH); + match url.map(|u| cache.get(ui.ctx(), u)) { + Some(Lookup::Ready(t)) => { + egui::Image::new(&t).uv(cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, rect); + }, + Some(Lookup::Loading) => icons::paint_spinner(ui, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), theme::FAINT), + _ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), Icon::User, theme::FAINT), + } +} + +/// The name under a tile (and a muted note after it). +fn caption(ui: &Ui, tile: Rect, name: &str, note: &str) { + let y = tile.bottom() + 15.0; + let galley = ui.painter().layout_no_wrap(name.to_owned(), FontId::new(13.0, theme::medium()), theme::INK); + let w = galley.size().x.min(tile.width()); + ui.painter().with_clip_rect(Rect::from_min_size(pos2(tile.left(), y - 10.0), vec2(tile.width(), 20.0))).galley(pos2(tile.left(), y - galley.size().y / 2.0), galley, theme::INK); + if !note.is_empty() { + ui.painter().text(pos2(tile.left() + w + 6.0, y), Align2::LEFT_CENTER, note, FontId::new(11.5, egui::FontFamily::Proportional), theme::MUTED); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use egui_kittest::Harness; + use egui_kittest::kittest::Queryable; + + struct State { + _rt: tokio::runtime::Runtime, + cache: MediaCache, + modal: CharactersModal, + characters: Vec, + actions: Vec, + fonts_ready: bool, + } + + fn harness() -> Harness<'static, State> { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().expect("runtime"); + let cache = MediaCache::new(rt.handle().clone(), reqwest::Client::new()); + let characters = vec![Character { token: "c1".into(), name: "Mira".into(), ..Default::default() }]; + let state = State { _rt: rt, cache, modal: CharactersModal::new(), characters, actions: Vec::new(), fonts_ready: false }; + let mut harness = Harness::builder().with_size(egui::vec2(1000.0, 800.0)).build_ui_state( + |ui, s: &mut State| { + // The harness draws one frame while building: fonts are installed then, used from the next. + if !s.fonts_ready { + crate::theme::apply(ui.ctx()); + s.fonts_ready = true; + return; + } + let actions = s.modal.show(ui.ctx(), &s.characters, &[], &mut s.cache); + s.actions.extend(actions.iter().map(|a| format!("{a:?}"))); + }, + state, + ); + harness.run(); + harness + } + + #[test] + fn picking_a_character_mentions_it() { + let mut harness = harness(); + harness.get_by_label("Mira").click(); + harness.run(); + assert!(harness.state().actions.iter().any(|a| a.contains("Select") && a.contains("Mira")), "{:?}", harness.state().actions); + } + + #[test] + fn creating_needs_a_name_and_a_reference() { + let mut harness = harness(); + harness.get_by_label("Create New").click(); + harness.run(); + harness.get_by_label("Create").click(); + harness.run(); + assert_eq!(harness.state().actions, [r#"Toast("Please enter a character name")"#]); + harness.get_by_role(egui::accesskit::Role::TextInput).focus(); + harness.run(); + harness.get_by_role(egui::accesskit::Role::TextInput).type_text("Mira"); + harness.run(); + harness.get_by_label("Create").click(); + harness.run(); + assert_eq!(harness.state().actions.last().map(String::as_str), Some(r#"Toast("Please upload a reference image")"#)); + } +} diff --git a/artcraft-native/src/overlays/library_picker.rs b/artcraft-native/src/overlays/library_picker.rs index b7c18cb3b3..9a0c06db6a 100644 --- a/artcraft-native/src/overlays/library_picker.rs +++ b/artcraft-native/src/overlays/library_picker.rs @@ -153,7 +153,7 @@ impl LibraryPicker { } fn picks(&self) -> Vec { - self.selected.iter().filter_map(|t| self.items.iter().find(|i| i.token == *t)).map(|f| RefMedia::from_library(self.kind, f.token.clone(), f.media_links.thumbnail(256), Some(f.media_links.cdn_url.clone()), f.maybe_duration_millis.map_or(0.0, |ms| ms as f32 / 1000.0))).collect() + self.selected.iter().filter_map(|t| self.items.iter().find(|i| i.token == *t)).map(|f| RefMedia::from_library(self.kind, f.token.clone(), f.thumbnail(256), Some(f.media_links.cdn_url.clone()), f.maybe_duration_millis.map_or(0.0, |ms| ms as f32 / 1000.0))).collect() } } @@ -162,7 +162,7 @@ fn tile_ui(ui: &mut egui::Ui, rect: Rect, item: &MediaFile, kind: RefKind, attac let resp = ui.interact(rect, Id::new(("picker-tile", &item.token)), if attached { Sense::hover() } else { Sense::click() }); let p = ui.painter(); p.rect_filled(rect, 0.0, theme::fade(theme::CONTROLS, 0.6)); - match item.media_links.thumbnail(256).map(|u| cache.get(ui.ctx(), &u)) { + match item.thumbnail(256).map(|u| cache.get(ui.ctx(), &u)) { Some(Lookup::Ready(t)) => { let image = egui::Image::new(&t).uv(cover_uv(t.size_vec2())); let image = if attached { image.tint(Color32::from_gray(90)) } else { image }; diff --git a/artcraft-native/src/overlays/lightbox.rs b/artcraft-native/src/overlays/lightbox.rs index d0dc197111..ba88e8bb31 100644 --- a/artcraft-native/src/overlays/lightbox.rs +++ b/artcraft-native/src/overlays/lightbox.rs @@ -1,23 +1,30 @@ -//! The lightbox (`lightbox-modal`): the media large on black, prev/next through the feed, and a -//! side column with the prompt, details and actions. +//! The lightbox (`lightbox-modal`): the media large on black (audio as a player panel), prev/next +//! through the feed, and a side column with the prompt, details and actions. use egui::{Align2, Color32, FontId, Id, Key, Rect, Sense, Ui, pos2, vec2}; +use crate::backend::audio::AudioPlayer; use crate::backend::media_cache::{Lookup, MediaCache}; +use crate::backend::video::{self, VideoPlayer}; use crate::backend::wire::Prompt; use crate::feed::grid::ModelNames; use crate::feed::types::{FeedItem, MediaKind}; use crate::theme; use crate::ui::icons::{self, Icon}; -use crate::ui::{creator_icons, widgets}; +use crate::ui::{creator_icons, waveform, widgets}; const SIDE_WIDTH: f32 = 280.0; +/// The audio panel's widest (`max-w-xl`) and its padding. +const AUDIO_PANEL_MAX: f32 = 576.0; +const AUDIO_PANEL_PAD: f32 = 24.0; pub struct Lightbox { pub kind: MediaKind, pub token: String, confirm_delete: bool, prompt_expanded: bool, + /// The in-app player for videos (when FFmpeg is installed). + player: Option, } #[derive(Debug, PartialEq)] @@ -35,10 +42,11 @@ pub enum LightboxAction { impl Lightbox { pub fn new(kind: MediaKind, token: String) -> Self { - Self { kind, token, confirm_delete: false, prompt_expanded: false } + Self { kind, token, confirm_delete: false, prompt_expanded: false, player: None } } - pub fn show(&mut self, ctx: &egui::Context, item: &FeedItem, order: &[String], prompt: Option<&Prompt>, names: &dyn ModelNames, cache: &mut MediaCache) -> Vec { + #[allow(clippy::too_many_arguments)] + pub fn show(&mut self, ctx: &egui::Context, item: &FeedItem, order: &[String], prompt: Option<&Prompt>, names: &dyn ModelNames, cache: &mut MediaCache, audio: &mut AudioPlayer) -> Vec { let mut actions = Vec::new(); let screen = ctx.content_rect(); let size = vec2((screen.width() - 80.0).min(1200.0), (screen.height() - 80.0).min(760.0)); @@ -62,7 +70,7 @@ impl Lightbox { ui.horizontal_top(|ui| { let media_rect = Rect::from_min_size(ui.cursor().min, vec2(ui.available_width() - SIDE_WIDTH - 16.0, size.y - 32.0)); ui.allocate_rect(media_rect, Sense::hover()); - self.media(ui, media_rect, item, cache, &prev, &next, &mut actions); + self.media(ui, media_rect, item, cache, audio, &prev, &next, &mut actions); ui.add_space(16.0); ui.vertical(|ui| { ui.set_width(SIDE_WIDTH); @@ -80,7 +88,29 @@ impl Lightbox { } #[allow(clippy::too_many_arguments)] - fn media(&self, ui: &mut Ui, rect: Rect, item: &FeedItem, cache: &mut MediaCache, prev: &Option, next: &Option, actions: &mut Vec) { + fn media(&mut self, ui: &mut Ui, rect: Rect, item: &FeedItem, cache: &mut MediaCache, audio: &mut AudioPlayer, prev: &Option, next: &Option, actions: &mut Vec) { + if item.kind == MediaKind::Audio { + self.player = None; + ui.painter().rect_filled(rect, 0.0, Color32::BLACK); + audio_panel(ui, rect, item, audio); + self.nav_buttons(ui, rect, prev, next, actions); + return; + } + if item.kind == MediaKind::Video && video::ffmpeg_available() { + if self.player.as_ref().is_none_or(|p| p.url() != item.full_url) { + self.player = Some(VideoPlayer::new(&item.full_url)); + } + let poster = match item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) { + Some(Lookup::Ready(t)) => Some(t), + _ => None, + }; + if let Some(player) = &mut self.player { + player.ui(ui, rect, poster.as_ref()); + } + self.nav_buttons(ui, rect, prev, next, actions); + return; + } + self.player = None; ui.painter().rect_filled(rect, 0.0, Color32::BLACK); let url = if item.kind == MediaKind::Image { Some(item.full_url.as_str()) } else { item.thumbnail.as_deref() }; match url.map(|u| cache.get(ui.ctx(), u)) { @@ -109,6 +139,11 @@ impl Lightbox { actions.push(LightboxAction::Play(item.full_url.clone())); } } + self.nav_buttons(ui, rect, prev, next, actions); + } + + /// Previous / next arrows at the media's sides (shown while hovering it). + fn nav_buttons(&self, ui: &mut Ui, rect: Rect, prev: &Option, next: &Option, actions: &mut Vec) { let hovering = ui.rect_contains_pointer(rect); for (target, left, tip) in [(prev, true, "Previous item"), (next, false, "Next item")] { let Some(token) = target else { @@ -214,7 +249,8 @@ impl Lightbox { ui.add_space(16.0); let w = (ui.available_width() - 8.0) / 2.0; - if item.prompt_token.is_some() && widgets::button(ui, Some(Icon::RotateCw), "Recreate", widgets::Kind::Primary, ui.available_width(), theme::CONTROL_H).clicked() { + // Audio can't be recreated yet (`Audio v1`). + if item.prompt_token.is_some() && item.kind != MediaKind::Audio && widgets::button(ui, Some(Icon::RotateCw), "Recreate", widgets::Kind::Primary, ui.available_width(), theme::CONTROL_H).clicked() { actions.push(LightboxAction::Recreate); } ui.horizontal_wrapped(|ui| { @@ -259,6 +295,17 @@ impl Lightbox { } } +/// Audio in the lightbox: a bordered panel with the music tile and the full-size player. +fn audio_panel(ui: &mut Ui, rect: Rect, item: &FeedItem, audio: &mut AudioPlayer) { + let width = (rect.width() - 48.0).min(AUDIO_PANEL_MAX); + let panel = Rect::from_center_size(rect.center(), vec2(width, AUDIO_PANEL_PAD * 2.0 + 64.0 + 20.0 + 44.0)); + ui.painter().rect(panel, theme::RADIUS, theme::fade(theme::CONTROLS, 0.6), theme::hairline(), egui::StrokeKind::Inside); + waveform::music_tile(ui, Rect::from_center_size(pos2(panel.center().x, panel.top() + AUDIO_PANEL_PAD + 32.0), vec2(64.0, 64.0)), false); + let bar = Rect::from_min_max(pos2(panel.left() + AUDIO_PANEL_PAD, panel.bottom() - AUDIO_PANEL_PAD - 44.0), pos2(panel.right() - AUDIO_PANEL_PAD, panel.bottom() - AUDIO_PANEL_PAD)); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar)); + waveform::player(&mut child, Id::new(("lightbox-player", &item.token)), audio, &item.full_url, item.duration_secs, false); +} + fn info_row(ui: &mut Ui, label: &str, value: impl FnOnce(&mut Ui)) { ui.horizontal(|ui| { let (rect, _) = ui.allocate_exact_size(vec2(96.0, 20.0), Sense::hover()); diff --git a/artcraft-native/src/overlays/mod.rs b/artcraft-native/src/overlays/mod.rs index 627ae6bd1c..bc637e6543 100644 --- a/artcraft-native/src/overlays/mod.rs +++ b/artcraft-native/src/overlays/mod.rs @@ -1,3 +1,4 @@ pub mod account; +pub mod characters; pub mod library_picker; pub mod lightbox; diff --git a/artcraft-native/src/pages/common.rs b/artcraft-native/src/pages/common.rs index 1d1f85d393..d52fa62769 100644 --- a/artcraft-native/src/pages/common.rs +++ b/artcraft-native/src/pages/common.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use egui::{Align2, Color32, Id, Mesh, Order, Rect, Ui, pos2, vec2}; use serde_json::{Map, Value}; +use crate::backend::audio::AudioPlayer; use crate::backend::media_cache::MediaCache; use crate::backend::{Backend, Modality, UploadedMedia}; use crate::feed::grid::{self, FeedAction, FeedView, RatioCache, ViewMode}; @@ -29,28 +30,55 @@ pub struct Env<'a> { pub ctx: egui::Context, pub backend: &'a Backend, pub cache: &'a mut MediaCache, + /// Plays reference clips and feed tracks. + pub audio: &'a mut AudioPlayer, pub toasts: &'a mut Toasts, pub catalog: &'a Catalog, pub signed_in: bool, pub enter_to_generate: bool, pub view_mode: ViewMode, + pub autoplay: bool, pub ratios: &'a mut RatioCache, pub prompts: &'a std::collections::HashMap, pub requests: &'a mut Vec, + /// The user's characters, newest first. + pub characters: &'a [crate::backend::wire::Character], } /// Things only the app can do. #[derive(Debug)] pub enum AppRequest { SignIn, - Open { kind: MediaKind, token: String }, - PickFromLibrary { kind: RefKind, slot: ImageSlot, max: usize, page: MediaKind }, - MakeVideo { token: String }, - Recreate { kind: MediaKind, token: String }, + Open { + kind: MediaKind, + token: String, + }, + PickFromLibrary { + kind: RefKind, + slot: ImageSlot, + max: usize, + page: MediaKind, + }, + MakeVideo { + token: String, + }, + Recreate { + kind: MediaKind, + token: String, + }, Share(String), Download(Vec), CopyText(String), Preview(String), + /// Play or stop an audio reference. + ToggleAudio { + ref_id: u64, + url: String, + }, + /// Open a file in the system's player. + PlayExternally(String), + /// Open the characters dialog (it mentions the picked one in the video prompt). + OpenCharacters, LoadMoreLibrary(FeedKey), } @@ -59,6 +87,7 @@ pub enum AppRequest { pub enum FeedKey { Image, Video, + Audio, Library, } @@ -67,6 +96,7 @@ impl FeedKey { match kind { MediaKind::Image => FeedKey::Image, MediaKind::Video => FeedKey::Video, + MediaKind::Audio => FeedKey::Audio, } } @@ -75,9 +105,33 @@ impl FeedKey { match self { FeedKey::Image => "image", FeedKey::Video => "video", - FeedKey::Library => "image,video", + FeedKey::Audio => "audio", + FeedKey::Library => "image,video,audio", } } + + /// What the feed's items default to (the library mixes kinds; each file says its own). + pub fn kind(self) -> MediaKind { + match self { + FeedKey::Image | FeedKey::Library => MediaKind::Image, + FeedKey::Video => MediaKind::Video, + FeedKey::Audio => MediaKind::Audio, + } + } + + /// The backend's name for this feed's library requests. + pub fn name(self) -> &'static str { + match self { + FeedKey::Image => "image", + FeedKey::Video => "video", + FeedKey::Audio => "audio", + FeedKey::Library => "library", + } + } + + pub fn from_name(name: &str) -> Option { + [FeedKey::Image, FeedKey::Video, FeedKey::Audio, FeedKey::Library].into_iter().find(|k| k.name() == name) + } } /// The page around the feed: the splash empty state (or the feed), a fade behind the prompt box, @@ -87,9 +141,9 @@ pub fn create_shell(ui: &mut Ui, env: &mut Env<'_>, kind: MediaKind, title: &str let area = ui.max_rect(); let bottom_offset = *box_height + 40.0; if feed.has_content() { - let view = FeedView { id: Id::new(("feed", kind.label())), mode: env.view_mode, pending: &feed.pending, failed: &feed.failed, items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: kind == MediaKind::Image, bottom_padding: bottom_offset + 24.0 }; + let view = FeedView { id: Id::new(("feed", kind.label())), mode: env.view_mode, pending: &feed.pending, failed: &feed.failed, items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: kind == MediaKind::Image, autoplay: env.autoplay, bottom_padding: bottom_offset + 24.0 }; let mut feed_ui = ui.new_child(egui::UiBuilder::new().max_rect(area.shrink2(vec2(12.0, 0.0)))); - let actions = grid::show(&mut feed_ui, &view, env.cache, env.ratios, env.catalog); + let actions = grid::show(&mut feed_ui, &view, env.cache, env.ratios, env.audio, env.catalog); handle_feed_actions(env, kind, feed, actions); bottom_fade(ui, area); } else { diff --git a/artcraft-native/src/pages/create_audio.rs b/artcraft-native/src/pages/create_audio.rs new file mode 100644 index 0000000000..e3fd677f7e --- /dev/null +++ b/artcraft-native/src/pages/create_audio.rs @@ -0,0 +1,882 @@ +//! Create Audio (`CreateAudio` + `PromptBoxAudio`): a song, sound or sample from a prompt, with +//! the model's options (style, instrumental, lyrics, loop, beat & key, tuning), audio or image +//! references, the cost, and the audio feed. + +use std::ops::RangeInclusive; +use std::path::PathBuf; + +use egui::{Align2, Color32, FontId, Id, Rect, Response, Sense, Stroke, TextureHandle, Ui, pos2, vec2}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; + +use crate::backend::media_cache::Lookup; +use crate::backend::{EnqueueMeta, Modality}; +use crate::feed::store::FeedStore; +use crate::feed::types::MediaKind; +use crate::models::{self, AudioCaps, Catalog, DEFAULT_AUDIO_MODEL, MUSICAL_KEYS, ModelInfo}; +use crate::pages::common::{self, AppRequest, CostState, Env}; +use crate::prompt_box::deck::cover_uv; +use crate::prompt_box::model_selector::model_selector; +use crate::prompt_box::pickers; +use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, RefStatus, References}; +use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot}; +use crate::theme; +use crate::ui::icons::{self, Icon}; +use crate::ui::toast::Toasts; +use crate::ui::widgets::{self, Leading}; + +const PLACEHOLDER: &str = "Describe the music or sound you want..."; +const STYLE_PLACEHOLDER: &str = "Style: e.g. dreamy synth-pop, female vocals"; +/// The audio references' total length cap (`AUDIO_REF_MAX_DURATION_SECONDS`). +const MAX_REF_SECS: f32 = 600.0; +/// Beat & Key and Tuning (`SoundsSettingsPopover`, `AudioTuningPopover`). +const BPM_RANGE: RangeInclusive = 40..=240; +const DEFAULT_BPM: u16 = 120; +const SPEED_RANGE: RangeInclusive = 0.5..=2.0; +const VOLUME_RANGE: RangeInclusive = 0.5..=2.0; +const PITCH_RANGE: RangeInclusive = -12..=12; +const POPOVER_WIDTH: f32 = 240.0; +/// Shown when generate fails on the server's side (`OMNI_GENERATE_OUTAGE_MESSAGE`). +const OUTAGE_MESSAGE: &str = "Couldn't start generation - the service is temporarily unavailable. Try again shortly."; + +/// What the page remembers between launches. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct AudioSettings { + pub prompt: String, + pub style_prompt: String, + pub model: Option, + pub instrumental: bool, + pub keep_lyrics: bool, + pub loopable: bool, + /// `None` = Auto. + pub bpm: Option, + pub musical_key: String, + /// `None` = the model's default. + pub sample_rate_hz: Option, + pub speed: f32, + pub volume: f32, + /// Semitones. + pub pitch: i8, +} + +impl Default for AudioSettings { + fn default() -> Self { + Self { prompt: String::new(), style_prompt: String::new(), model: None, instrumental: false, keep_lyrics: false, loopable: false, bpm: None, musical_key: "auto".to_owned(), sample_rate_hz: None, speed: 1.0, volume: 1.0, pitch: 0 } + } +} + +#[derive(Default)] +pub struct CreateAudioPage { + pub settings: AudioSettings, + pub refs: References, + pub feed: FeedStore, + pub cost: CostState, + pub generating: bool, + box_state: PromptBoxState, + box_height: f32, +} + +/// What the reference row asks for. +#[derive(Debug, PartialEq)] +enum RowAction { + AddAudio, + AudioFromLibrary, + AddImage, + Remove(u64), + Toggle(u64), +} + +impl CreateAudioPage { + pub fn new(settings: AudioSettings) -> Self { + let mut page = Self { settings, box_height: 138.0, ..Default::default() }; + page.box_state.editor.focus_requested = true; + page + } + + pub fn ui(&mut self, ui: &mut Ui, env: &mut Env<'_>) { + let model = common::selected_model(&env.catalog.audio_page, self.settings.model.as_deref(), DEFAULT_AUDIO_MODEL).cloned(); + if let Some(m) = &model { + let fields = self.fields(m, true); + self.cost.update(ui, env.backend, Modality::Audio, &fields); + } + let mut feed = std::mem::take(&mut self.feed); + let mut box_height = self.box_height; + common::create_shell(ui, env, MediaKind::Audio, "Create Audio", "Describe a song, a sound, or a sample", &mut feed, &mut box_height, |ui, env| self.prompt_box(ui, env, model.as_ref())); + self.feed = feed; + self.box_height = box_height; + } + + fn prompt_box(&mut self, ui: &mut Ui, env: &mut Env<'_>, model: Option<&ModelInfo>) { + let missing_ref = model.is_some_and(|m| self.missing_required_ref(m)); + let accepts: Vec = model.map_or_else(Vec::new, |m| [(RefKind::Audio, m.audio_refs_max), (RefKind::Image, m.image_refs_max)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect()); + let props = PromptBoxProps { id: Id::new("audio-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: None, deck: DeckMode::None, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !missing_ref, generating: self.generating, generate_tooltip: "Generate", warning: missing_ref.then_some("Audio track required"), banner: None, extra_input: !self.settings.style_prompt.is_empty() }; + + let playing = self.refs.audios.iter().map(|r| r.id).find(|id| env.audio.is_playing(&ref_key(*id))); + let image = self.refs.images.first().and_then(|r| r.preview.as_deref()).and_then(|key| match env.cache.get(&env.ctx, key) { + Lookup::Ready(t) => Some(t), + _ => None, + }); + let models = &env.catalog.audio_page; + let mut prompt = std::mem::take(&mut self.settings.prompt); + let settings = &mut self.settings; + let refs = &self.refs; + let mut picked_model = None; + let mut row_actions = Vec::new(); + let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) { + (Slot::ToolbarLeft, _) => { + if let Some(id) = model_selector(ui, Id::new("audio-model"), models, model, false) { + picked_model = Some(id); + } + if let Some(m) = model { + audio_toolbar(ui, &m.audio, settings); + } + }, + (Slot::AbovePrompt, Some(m)) => reference_row(ui, m, refs, playing, image.as_ref(), &mut row_actions), + (Slot::BelowPrompt, Some(m)) if m.audio.style_prompt => style_row(ui, &mut settings.style_prompt), + _ => {}, + }; + let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots); + self.settings.prompt = prompt; + if let Some(id) = picked_model { + self.settings.model = Some(id); + } + if let Some(m) = model { + for action in row_actions { + self.row_action(action, env, m); + } + } + for action in actions { + self.handle(action, env, model); + } + } + + fn handle(&mut self, action: PromptBoxAction, env: &mut Env<'_>, model: Option<&ModelInfo>) { + match action { + PromptBoxAction::Generate => self.generate(env, model), + // Toggles and tuning stay as they are. + PromptBoxAction::ClearAll => { + self.settings.prompt.clear(); + self.settings.style_prompt.clear(); + self.refs.clear(); + }, + PromptBoxAction::DroppedFiles(paths) => { + if let Some(m) = model { + self.add_dropped(env, m, &paths); + } + }, + PromptBoxAction::PastedImage(png) => match model { + Some(m) if m.image_refs_max > 0 => { + self.make_room_for_image(env); + common::upload_png(env, &mut self.refs, png, ImageSlot::Reference); + }, + _ => env.toasts.error("Image references aren't available here"), + }, + PromptBoxAction::Deck(_) | PromptBoxAction::MentionPicked(..) => {}, + } + } + + fn row_action(&mut self, action: RowAction, env: &mut Env<'_>, m: &ModelInfo) { + match action { + RowAction::AddAudio => { + let paths = common::pick_files(&[RefKind::Audio], m.audio_refs_max > 1); + self.add_audio(env, m, &paths); + }, + RowAction::AudioFromLibrary => { + let max = m.audio_refs_max.saturating_sub(self.refs.audios.len()).max(1); + env.requests.push(AppRequest::PickFromLibrary { kind: RefKind::Audio, slot: ImageSlot::Reference, max, page: MediaKind::Audio }); + }, + RowAction::AddImage => { + if let Some(path) = common::pick_files(&[RefKind::Image], false).first() { + self.make_room_for_image(env); + common::upload_path(env, &mut self.refs, path, RefKind::Image, ImageSlot::Reference); + } + }, + RowAction::Remove(id) => { + if env.audio.is_playing(&ref_key(id)) { + env.audio.stop(); + } + self.refs.remove(id); + }, + RowAction::Toggle(id) => { + if let Some(url) = self.refs.audios.iter().find(|r| r.id == id).and_then(|r| r.full_url.clone()) { + env.requests.push(AppRequest::ToggleAudio { ref_id: id, url }); + } + }, + } + } + + /// Dropped files (`usePromptBoxDrop`): audio wins over images; only the first image counts. + fn add_dropped(&mut self, env: &mut Env<'_>, m: &ModelInfo, paths: &[PathBuf]) { + let audios: Vec = paths.iter().filter(|p| RefKind::from_path(p) == Some(RefKind::Audio)).cloned().collect(); + let image = paths.iter().find(|p| RefKind::from_path(p) == Some(RefKind::Image)); + if !audios.is_empty() { + self.add_audio(env, m, &audios); + } else if let Some(path) = image { + if m.image_refs_max == 0 { + env.toasts.error("Image references aren't available here"); + return; + } + self.make_room_for_image(env); + common::upload_path(env, &mut self.refs, path, RefKind::Image, ImageSlot::Reference); + } else if !paths.is_empty() { + env.toasts.error(format!("Only {} files can be added here", accepted_kinds(m))); + } + } + + /// Uploads audio tracks into the free slots (dropping an image reference: they don't mix). + fn add_audio(&mut self, env: &mut Env<'_>, m: &ModelInfo, paths: &[PathBuf]) { + if paths.is_empty() { + return; + } + if m.audio_refs_max == 0 { + env.toasts.error("Audio references aren't available here"); + return; + } + let free = m.audio_refs_max.saturating_sub(self.refs.audios.len()); + if paths.len() > free { + env.toasts.error(format!("Max {} audio track{}", m.audio_refs_max, if m.audio_refs_max == 1 { "" } else { "s" })); + } + if free == 0 { + return; + } + self.make_room_for_audio(env.toasts); + for path in paths.iter().take(free) { + common::upload_path(env, &mut self.refs, path, RefKind::Audio, ImageSlot::Reference); + } + } + + /// Library picks, ready to attach: audio within the slots and the total-length cap. + pub fn attach_from_library(&mut self, toasts: &mut Toasts, catalog: &Catalog, kind: RefKind, picks: Vec) { + let Some(m) = common::selected_model(&catalog.audio_page, self.settings.model.as_deref(), DEFAULT_AUDIO_MODEL) else { + return; + }; + if kind != RefKind::Audio || picks.is_empty() { + common::attach_from_library(&mut self.refs, kind, ImageSlot::Reference, picks); + return; + } + self.make_room_for_audio(toasts); + let mut total = self.refs.total_secs(RefKind::Audio); + let mut accepted = Vec::new(); + for pick in picks.into_iter().take(m.audio_refs_max.saturating_sub(self.refs.audios.len())) { + if total + pick.duration_secs > MAX_REF_SECS { + toasts.error(duration_message()); + break; + } + total += pick.duration_secs; + accepted.push(pick); + } + common::attach_from_library(&mut self.refs, kind, ImageSlot::Reference, accepted); + } + + /// After an upload lands: keep the audio references under the total-length cap. + pub fn check_durations(&mut self, toasts: &mut Toasts, ref_id: u64) { + if self.refs.audios.iter().any(|r| r.id == ref_id) && self.refs.total_secs(RefKind::Audio) > MAX_REF_SECS + 0.5 { + self.refs.remove(ref_id); + toasts.error(duration_message()); + } + } + + /// Audio and image references don't mix: adding audio drops the image. + fn make_room_for_audio(&mut self, toasts: &mut Toasts) { + if !self.refs.images.is_empty() { + self.refs.images.clear(); + toasts.error("Removed image reference \u{2014} it can't be combined with audio"); + } + } + + /// One image at most, and never with audio. + fn make_room_for_image(&mut self, env: &mut Env<'_>) { + if !self.refs.audios.is_empty() { + if self.refs.audios.iter().any(|r| env.audio.is_playing(&ref_key(r.id))) { + env.audio.stop(); + } + self.refs.audios.clear(); + env.toasts.error("Removed audio reference \u{2014} it can't be combined with an image"); + } + self.refs.images.clear(); + } + + fn missing_required_ref(&self, m: &ModelInfo) -> bool { + m.audio.requires_audio_ref && self.refs.audios.len() != 1 + } + + fn generate(&mut self, env: &mut Env<'_>, model: Option<&ModelInfo>) { + let Some(m) = model.filter(|_| !self.generating && !self.settings.prompt.trim().is_empty()) else { + return; + }; + if self.missing_required_ref(m) { + env.toasts.error(format!("{} needs an audio track to work from \u{2014} add one first", m.name)); + return; + } + if !env.signed_in { + env.requests.push(AppRequest::SignIn); + return; + } + if self.refs.any_uploading() { + env.toasts.error("Please wait for your references to finish uploading"); + return; + } + self.generating = true; + let meta = EnqueueMeta { prompt: self.settings.prompt.trim().to_owned(), model_id: m.id.clone(), batch_count: 1, ref_image: None }; + env.backend.generate(Modality::Audio, self.fields(m, false), meta); + } + + pub fn on_enqueued(&mut self, env: &mut Env<'_>, job_tokens: &[String], meta: &EnqueueMeta) { + self.generating = false; + self.feed.add_enqueued(MediaKind::Audio, job_tokens, meta, env.catalog); + env.toasts.success("Audio generation enqueued!"); + } + + pub fn on_enqueue_failed(&mut self, env: &mut Env<'_>, message: &str, status: Option) { + self.generating = false; + env.toasts.error(failure_message(message, status)); + } + + /// The OmniGen request (`buildAudioGenerationBody`): only what the model takes, nothing null. + /// The estimate counts references with placeholders and needs no prompt. + fn fields(&self, m: &ModelInfo, estimate: bool) -> Map { + let (s, caps) = (&self.settings, &m.audio); + let mut f = Map::new(); + f.insert("model".into(), json!(m.id)); + let tokens = |list: &[RefMedia]| -> Vec { + if estimate { + vec!["placeholder".to_owned(); list.len()] + } else { + list.iter().filter_map(|r| r.token.clone()).collect() + } + }; + for (key, max, list) in [("audio_media_tokens", m.audio_refs_max, &self.refs.audios), ("image_media_tokens", m.image_refs_max, &self.refs.images)] { + let list = tokens(list.as_slice()); + if max > 0 && !list.is_empty() { + f.insert(key.into(), json!(list)); + } + } + if let Some(hz) = caps.sample_rate(s.sample_rate_hz).filter(|_| !caps.sample_rates.is_empty()) { + f.insert("sample_rate_hz".into(), json!(hz)); + } + if estimate { + return f; + } + let (prompt, style) = (s.prompt.trim(), s.style_prompt.trim()); + if !prompt.is_empty() { + f.insert("prompt".into(), json!(prompt)); + } + if caps.style_prompt && !style.is_empty() { + f.insert("style_prompt".into(), json!(style)); + } + for (supported, key, value) in [(caps.keep_lyrics, "keep_lyrics", s.keep_lyrics), (caps.instrumental, "is_instrumental", s.instrumental), (caps.loopable, "is_loopable", s.loopable)] { + if supported { + f.insert(key.into(), json!(value)); + } + } + if let Some(bpm) = s.bpm.filter(|_| caps.bpm) { + f.insert("bpm".into(), json!(bpm)); + } + if caps.musical_key { + f.insert("musical_key".into(), json!(musical_key(&s.musical_key).0)); + } + for (supported, key, value) in [(caps.speed, "speed", s.speed), (caps.volume, "volume", s.volume)] { + if supported { + f.insert(key.into(), json!(hundredths(value))); + } + } + if caps.pitch { + f.insert("pitch".into(), json!(s.pitch)); + } + f + } +} + +/// The audio pickers after the model (`PromptBoxAudio`'s toolbar): the toggles the model offers, +/// Beat & Key, and Tuning. +fn audio_toolbar(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) { + let toggles = [(caps.instrumental, &mut s.instrumental, "audio-instrumental", Icon::MicOff, "Instrumental"), (caps.keep_lyrics, &mut s.keep_lyrics, "audio-keep-lyrics", Icon::Mic, "Keep lyrics"), (caps.loopable, &mut s.loopable, "audio-loop", Icon::Repeat, "Loop")]; + for (supported, value, id, icon, label) in toggles { + if supported && pickers::toggle(ui, Id::new(id), icon, label, *value, &format!("{label}: {}", if *value { "ON" } else { "OFF" })) { + *value = !*value; + } + } + if caps.bpm || caps.musical_key { + beat_and_key(ui, caps, s); + } + if caps.has_tuning() { + tuning(ui, caps, s); + } +} + +/// The Beat & Key popover: a BPM slider with Auto, and the musical key grid. +fn beat_and_key(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) { + let mut parts = Vec::new(); + if caps.bpm { + parts.push(s.bpm.map_or_else(|| "Auto BPM".to_owned(), |b| format!("{b} BPM"))); + } + if caps.musical_key { + parts.push(musical_key(&s.musical_key).2.to_owned()); + } + let label = if parts.is_empty() { "Beat".to_owned() } else { parts.join(" \u{b7} ") }; + pickers::popover_pill(ui, Id::new("audio-beat"), Leading::Icon(Icon::Drum), &label, "Beat & Key", |ui| { + ui.set_width(POPOVER_WIDTH); + widgets::menu_header(ui, "Beat & Key"); + padded(ui, |ui| { + if caps.bpm { + ui.horizontal(|ui| { + setting_name(ui, "BPM"); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + setting_value(ui, &s.bpm.map_or_else(|| "Auto".to_owned(), |b| b.to_string())); + if s.bpm.is_some() && ui.add(egui::Button::new(egui::RichText::new("Auto").size(11.5).color(theme::ACCENT_INK)).frame(false)).on_hover_cursor(egui::CursorIcon::PointingHand).clicked() { + s.bpm = None; + } + }); + }); + let mut bpm = s.bpm.unwrap_or(DEFAULT_BPM); + if full_slider(ui, &mut bpm, BPM_RANGE, 1.0).changed() { + s.bpm = Some(bpm); + } + } + if caps.musical_key { + ui.add_space(6.0); + setting_name(ui, "Musical key"); + ui.spacing_mut().item_spacing = vec2(4.0, 4.0); + let width = ui.available_width(); + let (auto, keys) = MUSICAL_KEYS.split_first().expect("keys"); + if choice_chip(ui, auto.2, s.musical_key == auto.0, width, auto.1).clicked() { + s.musical_key = auto.0.to_owned(); + } + let cell = (width - 12.0) / 4.0; + for row in keys.chunks(4) { + ui.horizontal(|ui| { + for (value, label, short) in row { + if choice_chip(ui, short, s.musical_key == *value, cell, label).clicked() { + s.musical_key = (*value).to_owned(); + } + } + }); + } + } + }); + }); +} + +/// The Tuning popover: sample rate chips and the speed, volume and pitch sliders. +fn tuning(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) { + pickers::popover_pill(ui, Id::new("audio-tuning"), Leading::Icon(Icon::SlidersHorizontal), "Tuning", "Tuning", |ui| { + ui.set_width(POPOVER_WIDTH); + widgets::menu_header(ui, "Tuning"); + padded(ui, |ui| { + if !caps.sample_rates.is_empty() { + setting_name(ui, "Sample rate"); + let current = caps.sample_rate(s.sample_rate_hz); + ui.horizontal_wrapped(|ui| { + ui.spacing_mut().item_spacing = vec2(4.0, 4.0); + for hz in &caps.sample_rates { + let label = models::sample_rate_label(*hz); + if choice_chip(ui, &label, current == Some(*hz), 72.0, &label).clicked() { + s.sample_rate_hz = Some(*hz); + } + } + }); + ui.add_space(6.0); + } + if caps.speed { + labeled_slider(ui, "Speed", &format!("{:.2}\u{d7}", s.speed), &mut s.speed, SPEED_RANGE, 0.05); + } + if caps.volume { + labeled_slider(ui, "Volume", &format!("{:.2}\u{d7}", s.volume), &mut s.volume, VOLUME_RANGE, 0.05); + } + if caps.pitch { + labeled_slider(ui, "Pitch", &format!("{}{} st", if s.pitch > 0 { "+" } else { "" }, s.pitch), &mut s.pitch, PITCH_RANGE, 1.0); + } + }); + }); +} + +/// The references above the prompt (`AudioReferenceRow`): the audio tracks (play, remove), add or +/// pick more while there's room, and the image when the model takes one. +fn reference_row(ui: &mut Ui, m: &ModelInfo, refs: &References, playing: Option, image: Option<&TextureHandle>, actions: &mut Vec) { + if m.audio_refs_max == 0 && m.image_refs_max == 0 { + return; + } + ui.horizontal_wrapped(|ui| { + ui.spacing_mut().item_spacing = vec2(8.0, 6.0); + if m.audio_refs_max > 0 { + icons::icon(ui, Icon::Music, 14.0, theme::MUTED); + ui.label(egui::RichText::new(format!("Audio Track ({}/{})", refs.audios.len(), m.audio_refs_max)).size(12.5).color(theme::MUTED)); + if m.audio.requires_audio_ref && refs.audios.is_empty() { + ui.label(egui::RichText::new("required").size(12.5).color(theme::BAD)); + } + for (i, r) in refs.audios.iter().enumerate() { + audio_tile(ui, i, r, playing == Some(r.id), actions); + } + if refs.audios.len() < m.audio_refs_max { + let uploading = refs.audios.iter().any(|r| r.status == RefStatus::Uploading); + if ui.add_enabled_ui(!uploading, |ui| widgets::pill(ui, Leading::Icon(Icon::Upload), if uploading { "Uploading\u{2026}" } else { "Add audio" }, false)).inner.clicked() { + actions.push(RowAction::AddAudio); + } + if widgets::pill(ui, Leading::Icon(Icon::Folder), "From library", false).clicked() { + actions.push(RowAction::AudioFromLibrary); + } + } + } + if m.image_refs_max > 0 { + ui.add_space(8.0); + ui.label(egui::RichText::new("Image").size(12.5).color(theme::MUTED)); + match refs.images.first() { + Some(r) => image_tile(ui, r, image, actions), + None => { + if widgets::pill(ui, Leading::Icon(Icon::Image), "Add image", false).clicked() { + actions.push(RowAction::AddImage); + } + }, + } + } + }); + ui.add_space(8.0); + hairline(ui); + ui.add_space(8.0); +} + +/// One audio reference (`AudioRefTile`): play / stop, "Audio 1 · 12s", remove. +fn audio_tile(ui: &mut Ui, index: usize, r: &RefMedia, playing: bool, actions: &mut Vec) { + let uploading = r.status == RefStatus::Uploading; + let text = if uploading { format!("Audio {}", index + 1) } else { format!("Audio {} \u{b7} {}s", index + 1, r.duration_secs.round() as i64) }; + let galley = ui.painter().layout_no_wrap(text, FontId::new(12.5, theme::medium()), theme::INK); + let (rect, _) = ui.allocate_exact_size(vec2(6.0 + 24.0 + 8.0 + galley.size().x + 6.0 + 20.0 + 6.0, theme::CONTROL_H), Sense::hover()); + ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, Stroke::new(1.0, theme::LINE), egui::StrokeKind::Inside); + let play = Rect::from_center_size(pos2(rect.left() + 18.0, rect.center().y), vec2(24.0, 24.0)); + if uploading { + icons::paint_spinner(ui, play.shrink(4.0), theme::MUTED); + } else { + let resp = ui.interact(play, Id::new(("audio-ref-play", r.id)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand); + let label = if playing { "Stop" } else { "Play" }; + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label)); + let fill = match (playing, resp.hovered()) { + (true, _) => theme::ACCENT, + (false, true) => theme::WASH_HOVER, + (false, false) => theme::WASH, + }; + ui.painter().circle_filled(play.center(), 12.0, fill); + if playing { + ui.painter().rect_filled(Rect::from_center_size(play.center(), vec2(8.0, 8.0)), 1.0, Color32::WHITE); + } else { + icons::paint(ui.painter(), play.shrink(7.0).translate(vec2(1.0, 0.0)), Icon::Play, theme::INK); + } + if resp.clicked() { + actions.push(RowAction::Toggle(r.id)); + } + } + ui.painter().galley(pos2(play.right() + 8.0, rect.center().y - galley.size().y / 2.0), galley, theme::INK); + let remove = Rect::from_center_size(pos2(rect.right() - 16.0, rect.center().y), vec2(20.0, 20.0)); + if x_button(ui, remove, Id::new(("audio-ref-remove", r.id)), "Remove audio", true) { + actions.push(RowAction::Remove(r.id)); + } +} + +/// The image reference: a 36 px thumbnail with a remove × on hover. +fn image_tile(ui: &mut Ui, r: &RefMedia, texture: Option<&TextureHandle>, actions: &mut Vec) { + let (rect, _) = ui.allocate_exact_size(vec2(36.0, 36.0), Sense::hover()); + ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, Stroke::new(1.0, theme::LINE), egui::StrokeKind::Inside); + if let Some(t) = texture { + egui::Image::new(t).uv(cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, rect.shrink(1.0)); + } + if r.status == RefStatus::Uploading { + icons::paint_spinner(ui, rect.shrink(9.0), Color32::WHITE); + } + let remove = Rect::from_min_size(pos2(rect.right() - 14.0, rect.top() - 4.0), vec2(18.0, 18.0)); + let shown = ui.rect_contains_pointer(rect.union(remove)); + if x_button(ui, remove, Id::new(("image-ref-remove", r.id)), "Remove image", shown) { + actions.push(RowAction::Remove(r.id)); + } +} + +/// The style line under the prompt (`StylePromptRow`). +fn style_row(ui: &mut Ui, style: &mut String) { + ui.add_space(6.0); + hairline(ui); + ui.add_space(6.0); + ui.horizontal(|ui| { + icons::icon(ui, Icon::Tags, 14.0, theme::DIM); + let clear_w = if style.is_empty() { 0.0 } else { 28.0 }; + ui.add(egui::TextEdit::singleline(style).id_salt("audio-style").hint_text(STYLE_PLACEHOLDER).frame(egui::Frame::NONE).font(FontId::new(14.0, egui::FontFamily::Proportional)).desired_width(ui.available_width() - clear_w)); + if !style.is_empty() { + let (rect, _) = ui.allocate_exact_size(vec2(20.0, 20.0), Sense::hover()); + if x_button(ui, rect, Id::new("audio-style-clear"), "Clear style", true) { + style.clear(); + } + } + }); +} + +/// A small round × at `rect`, painted while `shown` (it stays clickable for keyboards and tests). +fn x_button(ui: &mut Ui, rect: Rect, id: Id, label: &str, shown: bool) -> bool { + let resp = ui.interact(rect, id, Sense::click()); + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label)); + if shown || resp.hovered() { + let fill = if resp.hovered() { theme::fade(theme::DANGER, 0.7) } else { Color32::from_black_alpha(128) }; + ui.painter().circle_filled(rect.center(), rect.width() / 2.0, fill); + icons::paint(ui.painter(), rect.shrink(rect.width() * 0.27), Icon::X, Color32::WHITE); + } + resp.on_hover_text(label).on_hover_cursor(egui::CursorIcon::PointingHand).clicked() +} + +/// A selectable chip in the popovers (keys, sample rates): outlined, filled when selected. +fn choice_chip(ui: &mut Ui, label: &str, selected: bool, width: f32, tooltip: &str) -> Response { + let (rect, resp) = ui.allocate_exact_size(vec2(width, 28.0), Sense::click()); + let (fill, border) = match (selected, resp.hovered()) { + (true, _) => (theme::WASH_HOVER, theme::fade(Color32::WHITE, 0.8)), + (false, true) => (theme::WASH, theme::LINE_STRONG), + (false, false) => (Color32::TRANSPARENT, theme::LINE), + }; + ui.painter().rect(rect, theme::RADIUS, fill, Stroke::new(1.0, border), egui::StrokeKind::Inside); + ui.painter().text(rect.center(), Align2::CENTER_CENTER, label, FontId::new(12.5, theme::medium()), if selected { theme::INK } else { theme::MUTED }); + resp.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Button, true, selected, label)); + resp.on_hover_text(tooltip).on_hover_cursor(egui::CursorIcon::PointingHand) +} + +/// A setting's name, its value at the right, and a slider across the popover. +fn labeled_slider(ui: &mut Ui, name: &str, value: &str, number: &mut N, range: RangeInclusive, step: f64) { + ui.horizontal(|ui| { + setting_name(ui, name); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| setting_value(ui, value)); + }); + full_slider(ui, number, range, step); + ui.add_space(4.0); +} + +fn full_slider(ui: &mut Ui, number: &mut N, range: RangeInclusive, step: f64) -> Response { + ui.style_mut().spacing.slider_width = ui.available_width(); + ui.add(egui::Slider::new(number, range).step_by(step).show_value(false)) +} + +fn setting_name(ui: &mut Ui, name: &str) { + ui.label(egui::RichText::new(name).size(12.0).color(theme::MUTED)); +} + +fn setting_value(ui: &mut Ui, value: &str) { + ui.label(egui::RichText::new(value).size(12.0).family(theme::semibold()).color(theme::INK)); +} + +fn padded(ui: &mut Ui, content: impl FnOnce(&mut Ui)) { + egui::Frame::new().inner_margin(egui::Margin::symmetric(8, 6)).show(ui, content); +} + +fn hairline(ui: &mut Ui) { + let (line, _) = ui.allocate_exact_size(vec2(ui.available_width(), 1.0), Sense::hover()); + ui.painter().rect_filled(line, 0.0, theme::fade(Color32::WHITE, 0.1)); +} + +/// The key a stored value names (Auto for anything unknown): value, label, short label. +fn musical_key(value: &str) -> &'static (&'static str, &'static str, &'static str) { + MUSICAL_KEYS.iter().find(|k| k.0 == value).unwrap_or(&MUSICAL_KEYS[0]) +} + +/// The audio player's key for a reference clip. +fn ref_key(id: u64) -> String { + format!("ref:{id}") +} + +/// A slider's value to two decimals (0.05 steps drift in `f32`). +fn hundredths(value: f32) -> f64 { + (f64::from(value) * 100.0).round() / 100.0 +} + +/// "audio", "audio or image" (for the drop toasts). +fn accepted_kinds(m: &ModelInfo) -> String { + let kinds: Vec<&str> = [("audio", m.audio_refs_max), ("image", m.image_refs_max)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect(); + if kinds.is_empty() { + "reference".to_owned() + } else { + kinds.join(" or ") + } +} + +fn duration_message() -> String { + format!("Total audio duration cannot exceed {}s", MAX_REF_SECS as u32) +} + +/// What a failed generate tells the user: out of credits, an outage, or the server's message. +fn failure_message(message: &str, status: Option) -> String { + match status { + Some(402) => "Not enough credits for this generation".to_owned(), + Some(code) if code >= 500 => OUTAGE_MESSAGE.to_owned(), + _ if message.is_empty() => "Failed to start audio generation".to_owned(), + _ => message.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn model(json: Value) -> ModelInfo { + models::audio_model(&serde_json::from_value(json).expect("audio model")) + } + + /// The server's audio models (`configs/omni_gen/audio_models.rs`). + fn suno_music() -> ModelInfo { + model(json!({ "model": "suno_music", "style_prompt_supported": true, "instrumental_toggle_supported": true })) + } + + fn suno_sounds() -> ModelInfo { + model(json!({ "model": "suno_sounds", "loopable_toggle_supported": true, "bpm_supported": true, "musical_key_supported": true })) + } + + fn suno_remix() -> ModelInfo { + model(json!({ "model": "suno_remix", "style_prompt_supported": true, "keep_lyrics_supported": true, "audio_references_supported": true, "audio_references_max": 1 })) + } + + fn seed_audio() -> ModelInfo { + model(json!({ "model": "seed_audio_1p0", "audio_references_supported": true, "audio_references_max": 3, "image_references_supported": true, "image_references_max": 1, "sample_rate_hz_options": [8000, 16000, 24000, 32000, 44100, 48000], "sample_rate_hz_default": 24000, "speed_supported": true, "volume_supported": true, "pitch_supported": true })) + } + + fn page(settings: AudioSettings) -> CreateAudioPage { + CreateAudioPage { settings, ..Default::default() } + } + + fn audio_ref(token: &str, secs: f32) -> RefMedia { + RefMedia::from_library(RefKind::Audio, token.into(), None, Some(format!("https://cdn/{token}.mp3")), secs) + } + + mod request_bodies { + use super::*; + + #[test] + fn suno_music_sends_only_what_it_takes() { + let p = page(AudioSettings { prompt: " lofi beat ".into(), style_prompt: " chill ".into(), instrumental: true, keep_lyrics: true, bpm: Some(90), ..Default::default() }); + assert_eq!(Value::Object(p.fields(&suno_music(), false)), json!({ "model": "suno_music", "prompt": "lofi beat", "style_prompt": "chill", "is_instrumental": true })); + } + + #[test] + fn suno_sounds_sends_its_toggles_beat_and_key() { + let mut p = page(AudioSettings { prompt: "door creak".into(), ..Default::default() }); + assert_eq!(Value::Object(p.fields(&suno_sounds(), false)), json!({ "model": "suno_sounds", "prompt": "door creak", "is_loopable": false, "musical_key": "auto" }), "Auto BPM is left to the model"); + p.settings.bpm = Some(128); + p.settings.musical_key = "c_minor".into(); + let f = p.fields(&suno_sounds(), false); + assert_eq!((f["bpm"].clone(), f["musical_key"].clone()), (json!(128), json!("c_minor"))); + } + + #[test] + fn seed_audio_sends_references_and_tuning() { + let mut p = page(AudioSettings { prompt: "rain".into(), speed: 1.05, volume: 0.5, pitch: -3, sample_rate_hz: Some(11025), ..Default::default() }); + p.refs.audios = vec![audio_ref("mf_a", 10.0), audio_ref("mf_b", 20.0)]; + assert_eq!(Value::Object(p.fields(&seed_audio(), false)), json!({ "model": "seed_audio_1p0", "prompt": "rain", "audio_media_tokens": ["mf_a", "mf_b"], "sample_rate_hz": 24000, "speed": 1.05, "volume": 0.5, "pitch": -3 })); + } + + #[test] + fn estimates_count_references_without_the_prompt() { + let mut p = page(AudioSettings { prompt: "rain".into(), sample_rate_hz: Some(44100), ..Default::default() }); + p.refs.audios = vec![RefMedia::uploading(RefKind::Audio, None), audio_ref("mf_b", 5.0)]; + assert_eq!(Value::Object(p.fields(&seed_audio(), true)), json!({ "model": "seed_audio_1p0", "audio_media_tokens": ["placeholder", "placeholder"], "sample_rate_hz": 44100 })); + assert_eq!(Value::Object(p.fields(&suno_music(), true)), json!({ "model": "suno_music" }), "references the model can't take don't count"); + } + } + + mod rules { + use super::*; + + #[test] + fn remixes_need_exactly_one_track() { + let mut p = page(AudioSettings::default()); + assert!(p.missing_required_ref(&suno_remix())); + assert!(!p.missing_required_ref(&suno_music())); + p.refs.audios.push(audio_ref("mf_a", 30.0)); + assert!(!p.missing_required_ref(&suno_remix())); + p.refs.audios.push(audio_ref("mf_b", 30.0)); + assert!(p.missing_required_ref(&suno_remix()), "two tracks are too many"); + } + + #[test] + fn library_picks_respect_slots_the_cap_and_drop_the_image() { + let mut catalog = Catalog::default(); + catalog.set_audio(vec![seed_audio()]); + let mut p = page(AudioSettings { model: Some("seed_audio_1p0".into()), ..Default::default() }); + p.refs.images.push(RefMedia::from_library(RefKind::Image, "mf_img".into(), None, None, 0.0)); + let mut toasts = Toasts::default(); + p.attach_from_library(&mut toasts, &catalog, RefKind::Audio, vec![audio_ref("mf_a", 400.0), audio_ref("mf_b", 300.0), audio_ref("mf_c", 10.0)]); + let tokens: Vec<_> = p.refs.audios.iter().filter_map(|r| r.token.as_deref()).collect(); + assert_eq!(tokens, ["mf_a"], "the second would pass 600s, so picking stops there"); + assert!(p.refs.images.is_empty(), "audio and an image don't mix"); + } + + #[test] + fn uploads_past_the_cap_are_dropped() { + let mut p = page(AudioSettings::default()); + p.refs.audios = vec![audio_ref("mf_a", 590.0), audio_ref("mf_b", 30.0)]; + let late = p.refs.audios[1].id; + p.check_durations(&mut Toasts::default(), late); + assert_eq!(p.refs.audios.len(), 1); + } + + #[test] + fn failures_explain_credits_and_outages() { + assert_eq!(failure_message("Payment required", Some(402)), "Not enough credits for this generation"); + assert_eq!(failure_message("boom", Some(503)), OUTAGE_MESSAGE); + assert_eq!(failure_message("Prompt too long", Some(400)), "Prompt too long"); + assert_eq!(failure_message("", None), "Failed to start audio generation"); + } + } + + /// The toolbar headlessly (egui_kittest): toggles, Beat & Key and Tuning. + mod toolbar { + use egui_kittest::Harness; + use egui_kittest::kittest::{NodeT, Queryable}; + + use super::*; + + struct State { + caps: AudioCaps, + settings: AudioSettings, + fonts_ready: bool, + } + + #[test] + fn toggles_show_per_model_and_flip() { + let mut harness = build(suno_music().audio); + assert!(harness.query_by_label("Loop").is_none(), "Suno Music has no loop toggle"); + harness.get_by_label("Instrumental").click(); + harness.run(); + assert!(harness.state().settings.instrumental); + } + + #[test] + fn beat_and_key_sets_the_key_and_bpm() { + let mut harness = build(suno_sounds().audio); + harness.get_by_label("Auto BPM \u{b7} Auto").click(); + harness.run(); + harness.get_by_label("Cm").click(); + harness.run(); + assert_eq!(harness.state().settings.musical_key, "c_minor"); + assert!(harness.query_by_label("Auto BPM \u{b7} Cm").is_some(), "the pill shows the key"); + } + + #[test] + fn tuning_picks_a_sample_rate() { + let mut harness = build(seed_audio().audio); + harness.get_by_label("Tuning").click(); + harness.run(); + // egui reports a selected button as toggled. + assert!(harness.query_by_label("24 kHz").is_some_and(|n| n.accesskit_node().toggled() == Some(egui::accesskit::Toggled::True)), "the default is selected"); + harness.get_by_label("44.1 kHz").click(); + harness.run(); + assert_eq!(harness.state().settings.sample_rate_hz, Some(44100)); + } + + fn build(caps: AudioCaps) -> Harness<'static, State> { + let mut harness = Harness::builder().with_size(egui::vec2(1000.0, 600.0)).build_ui_state( + |ui, s: &mut State| { + if !s.fonts_ready { + crate::theme::apply(ui.ctx()); + s.fonts_ready = true; + return; + } + ui.add_space(400.0); + ui.horizontal(|ui| audio_toolbar(ui, &s.caps, &mut s.settings)); + }, + State { caps, settings: AudioSettings::default(), fonts_ready: false }, + ); + harness.run(); + harness + } + } +} diff --git a/artcraft-native/src/pages/create_image.rs b/artcraft-native/src/pages/create_image.rs index 74fb0f5800..8be8bf7658 100644 --- a/artcraft-native/src/pages/create_image.rs +++ b/artcraft-native/src/pages/create_image.rs @@ -15,7 +15,7 @@ use crate::pages::toolbar; use crate::prompt_box::deck::{DeckAction, DeckLimits}; use crate::prompt_box::model_selector::model_selector; use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, References}; -use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, editor}; +use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot, editor}; const PLACEHOLDER: &str = "Describe what you want in the image..."; @@ -69,38 +69,41 @@ impl CreateImagePage { let deck = if max_images > 0 { DeckMode::References(DeckLimits { max_images, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: env.signed_in }) } else { DeckMode::None }; let accepts: &[RefKind] = if max_images > 0 { &[RefKind::Image] } else { &[] }; let prompt_max = model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max); - let props = PromptBoxProps { id: Id::new("image-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: Some(prompt_max), deck, accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some(), generating: self.generating, generate_tooltip: "Generate", warning: None, banner: None }; + let props = PromptBoxProps { id: Id::new("image-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: Some(prompt_max), deck, accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some(), generating: self.generating, generate_tooltip: "Generate", warning: None, banner: None, extra_input: false }; let models = &env.catalog.image_page; let settings = &mut self.settings; - let mut picked_model = None; - let mut left = |ui: &mut Ui| { - if let Some(id) = model_selector(ui, Id::new("image-model"), models, model, false) { - picked_model = Some(id); - } - let Some(m) = model else { - return; - }; - let aspect = ModelInfo::resolve(settings.aspect_ratio.as_deref(), &m.aspect_ratios, m.aspect_default.as_ref()); - if let Some(v) = toolbar::aspect_ratio(ui, Id::new("image-aspect"), m, aspect.as_deref().unwrap_or("")) { - settings.aspect_ratio = Some(v); - } - let resolution = ModelInfo::resolve(settings.resolution.as_deref(), &m.resolutions, m.resolution_default.as_ref()); - if let Some(v) = toolbar::resolution(ui, Id::new("image-resolution"), m, resolution.as_deref().unwrap_or("")) { - settings.resolution = Some(v); - } - let quality = ModelInfo::resolve(settings.quality.as_deref(), &m.qualities, m.quality_default.as_ref()); - if let Some(v) = toolbar::quality(ui, Id::new("image-quality"), m, quality.as_deref().unwrap_or("")) { - settings.quality = Some(v); - } - }; let count = settings.count; + let mut picked_model = None; let mut picked_count = None; - let mut right = |ui: &mut Ui| { - if let Some(m) = model { - picked_count = toolbar::count(ui, Id::new("image-count"), m, count, "images"); - } + let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) { + (Slot::ToolbarLeft, _) => { + if let Some(id) = model_selector(ui, Id::new("image-model"), models, model, false) { + picked_model = Some(id); + } + let Some(m) = model else { + return; + }; + let aspect = ModelInfo::resolve(settings.aspect_ratio.as_deref(), &m.aspect_ratios, m.aspect_default.as_ref()); + if let Some(v) = toolbar::aspect_ratio(ui, Id::new("image-aspect"), m, aspect.as_deref().unwrap_or("")) { + settings.aspect_ratio = Some(v); + } + let resolution = ModelInfo::resolve(settings.resolution.as_deref(), &m.resolutions, m.resolution_default.as_ref()); + if let Some(v) = toolbar::resolution(ui, Id::new("image-resolution"), m, resolution.as_deref().unwrap_or("")) { + settings.resolution = Some(v); + } + let quality = ModelInfo::resolve(settings.quality.as_deref(), &m.qualities, m.quality_default.as_ref()); + if let Some(v) = toolbar::quality(ui, Id::new("image-quality"), m, quality.as_deref().unwrap_or("")) { + settings.quality = Some(v); + } + }, + (Slot::ToolbarRight, Some(m)) => { + if let Some(n) = toolbar::count(ui, Id::new("image-count"), m, count, "images") { + picked_count = Some(n); + } + }, + _ => {}, }; - let actions = prompt_box::show(ui, &mut settings.prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut left, &mut right); + let actions = prompt_box::show(ui, &mut settings.prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots); if let Some(id) = picked_model { self.settings.model = Some(id); } @@ -143,6 +146,7 @@ impl CreateImagePage { DeckAction::SwapFrames => {}, }, PromptBoxAction::DroppedFiles(paths) => self.add_dropped(env, &paths, max_images), + PromptBoxAction::MentionPicked(..) => {}, PromptBoxAction::PastedImage(png) => { if self.refs.images.len() >= max_images { env.toasts.error(max_images_message(max_images)); diff --git a/artcraft-native/src/pages/create_video.rs b/artcraft-native/src/pages/create_video.rs index ef0f76a614..7155f985d3 100644 --- a/artcraft-native/src/pages/create_video.rs +++ b/artcraft-native/src/pages/create_video.rs @@ -20,7 +20,7 @@ use crate::prompt_box::editor::{self, MentionItem, MentionKind}; use crate::prompt_box::model_selector::model_selector; use crate::prompt_box::pickers::{self, PickOption, Trigger}; use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, References}; -use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState}; +use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot}; use crate::ui::icons::Icon; use crate::ui::widgets::Leading; @@ -67,6 +67,8 @@ pub struct CreateVideoPage { pub generating: bool, box_state: PromptBoxState, box_height: f32, + /// Which character each mentioned name means, when the user picked one explicitly. + character_picks: std::collections::HashMap, } impl CreateVideoPage { @@ -104,10 +106,14 @@ impl CreateVideoPage { (DeckMode::References(l), _) => [(RefKind::Image, l.max_images), (RefKind::Video, l.max_videos), (RefKind::Audio, l.max_audios)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect(), _ => Vec::new(), }; - let mentions = if reference { self.mentions() } else { Vec::new() }; + let characters_on = model.is_some_and(supports_characters); + let mut mentions = if reference { self.mentions() } else { Vec::new() }; + if characters_on { + mentions.extend(editor::character_mentions(env.characters.iter().map(|c| (c.name.as_str(), c.token.as_str(), c.avatar_url())))); + } let needs_start = model.is_some_and(|m| self.needs_start_frame(m)); let banner = model.filter(|m| !m.text_to_video && !self.has_image_input()).map(|_| TEXT_ONLY_BANNER); - let props = PromptBoxProps { id: Id::new("video-prompt-box"), placeholder: if reference { PLACEHOLDER_REFERENCE } else { PLACEHOLDER }, enter_to_generate: env.enter_to_generate, mentions: &mentions, max_length: Some(model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max)), deck, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !needs_start, generating: self.generating, generate_tooltip: if needs_start { "Add a starting image before generating" } else { "Generate" }, warning: needs_start.then_some("Starting frame required"), banner }; + let props = PromptBoxProps { id: Id::new("video-prompt-box"), placeholder: if reference { PLACEHOLDER_REFERENCE } else { PLACEHOLDER }, enter_to_generate: env.enter_to_generate, mentions: &mentions, max_length: Some(model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max)), deck, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !needs_start, generating: self.generating, generate_tooltip: if needs_start { "Add a starting image before generating" } else { "Generate" }, warning: needs_start.then_some("Starting frame required"), banner, extra_input: false }; let models = &env.catalog.video_page; let mut prompt = std::mem::take(&mut self.settings.prompt); @@ -116,21 +122,28 @@ impl CreateVideoPage { let with_refs = reference && !self.refs.images.is_empty(); let mut picked_model = None; let mut picked_mode = None; - let mut left = |ui: &mut Ui| { - if let Some(id) = model_selector(ui, Id::new("video-model"), models, model, true) { - picked_model = Some(id); - } - if let Some(m) = model { - picked_mode = video_toolbar(ui, m, settings, with_refs).or(picked_mode); - } - }; let mut picked_count = None; - let mut right = |ui: &mut Ui| { - if let Some(m) = model { - picked_count = toolbar::count(ui, Id::new("video-count"), m, count, "videos"); - } + let mut open_characters = false; + let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) { + (Slot::ToolbarLeft, _) => { + if let Some(id) = model_selector(ui, Id::new("video-model"), models, model, true) { + picked_model = Some(id); + } + if let Some(m) = model { + picked_mode = video_toolbar(ui, m, settings, with_refs).or(picked_mode); + if characters_on && pickers::toggle(ui, Id::new("video-characters"), Icon::User, "@Characters", false, "Characters") { + open_characters = true; + } + } + }, + (Slot::ToolbarRight, Some(m)) => { + if let Some(n) = toolbar::count(ui, Id::new("video-count"), m, count, "videos") { + picked_count = Some(n); + } + }, + _ => {}, }; - let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut left, &mut right); + let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots); self.settings.prompt = prompt; if let Some(id) = picked_model { self.settings.model = Some(id); @@ -141,6 +154,9 @@ impl CreateVideoPage { if let Some(mode) = picked_mode { switch_mode(&mut self.settings, &mut self.refs, mode); } + if open_characters { + env.requests.push(AppRequest::OpenCharacters); + } for action in actions { self.handle(action, env, model, limits.as_ref()); } @@ -176,14 +192,27 @@ impl CreateVideoPage { let item = self.refs.images.remove(from); self.refs.images.insert(to.min(self.refs.images.len()), item); }, - DeckAction::Preview(id) => { - if let Some(url) = self.refs.find_mut(id).and_then(|r| r.full_url.clone().or_else(|| r.preview.clone())) { - env.requests.push(AppRequest::Preview(url)); - } + DeckAction::Preview(id) => match self.refs.find_mut(id) { + // Audio cards play and stop on click (the webapp's DeckCard). + Some(r) if r.kind == RefKind::Audio => { + if let Some(url) = r.full_url.clone() { + env.requests.push(AppRequest::ToggleAudio { ref_id: id, url }); + } + }, + Some(r) => { + if let Some(url) = r.full_url.clone().or_else(|| r.preview.clone()) { + env.requests.push(if r.kind == RefKind::Video { AppRequest::PlayExternally(url) } else { AppRequest::Preview(url) }); + } + }, + None => {}, }, DeckAction::SwapFrames => self.refs.swap_frames(), }, PromptBoxAction::DroppedFiles(paths) => self.add_files(env, &paths, ImageSlot::Reference, limits), + PromptBoxAction::MentionPicked(label, Some(token)) => { + self.character_picks.insert(label.trim_start_matches('@').to_owned(), token); + }, + PromptBoxAction::MentionPicked(_, None) => {}, PromptBoxAction::PastedImage(png) => match self.settings.input_mode { InputMode::Keyframe => match self.free_keyframe(model) { Some(slot) => common::upload_png(env, &mut self.refs, png, slot), @@ -288,7 +317,27 @@ impl CreateVideoPage { self.generating = true; let ref_image = self.refs.first_frame.as_ref().or(self.refs.images.first()).and_then(|r| r.preview.clone()); let meta = EnqueueMeta { prompt: self.settings.prompt.clone(), model_id: model.id.clone(), batch_count: u32::from(model.valid_batch(self.settings.count)), ref_image }; - env.backend.generate(Modality::Video, self.fields(model, false), meta); + let mut fields = self.fields(model, false); + if supports_characters(model) { + let tokens = mentioned_characters(&self.settings.prompt, env.characters, &self.character_picks); + if !tokens.is_empty() { + fields.insert("reference_character_tokens".into(), json!(tokens)); + } + } + env.backend.generate(Modality::Video, fields, meta); + } + + /// The characters dialog picked `name`: mention it at the end of the prompt. + pub fn mention_character(&mut self, name: &str, token: &str) { + let p = &mut self.settings.prompt; + if !p.is_empty() && !p.ends_with(char::is_whitespace) { + p.push(' '); + } + p.push('@'); + p.push_str(name); + p.push(' '); + self.character_picks.insert(name.to_owned(), token.to_owned()); + self.box_state.editor.focus_requested = true; } /// The OmniGen request (`GenerateVideoRequest` after `api_fields`). @@ -433,7 +482,7 @@ impl CreateVideoPage { "audioref" => RefKind::Audio, _ => RefKind::Image, }; - let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256), Some(c.media_links.cdn_url.clone()), 0.0); + let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256).filter(|_| kind != RefKind::Audio), Some(c.media_links.cdn_url.clone()), 0.0); match (c.semantic.as_str(), kind) { ("vid_end_frame", _) => self.refs.last_frame = Some(item), (_, RefKind::Image) if s.input_mode == InputMode::Keyframe && self.refs.first_frame.is_none() => self.refs.first_frame = Some(item), @@ -506,6 +555,33 @@ fn switch_mode(s: &mut VideoSettings, refs: &mut References, mode: InputMode) { s.input_mode = mode; } +/// Whether the model takes `@Character` mentions (the listing says so; Seedance 2.0 always does). +fn supports_characters(m: &ModelInfo) -> bool { + m.character_refs_max > 0 || m.id == "seedance_2p0" +} + +/// One token per character named in the prompt (`@Name`, not inside a longer word): the user's +/// explicit pick for that name, else the newest character with it. +fn mentioned_characters(prompt: &str, characters: &[crate::backend::wire::Character], picks: &std::collections::HashMap) -> Vec { + let mut names: Vec<&str> = characters.iter().map(|c| c.name.as_str()).filter(|n| !n.is_empty()).collect(); + names.sort_by_key(|n| std::cmp::Reverse(n.len())); + names.dedup(); + let mut tokens = Vec::new(); + for name in names { + let needle = format!("@{name}"); + let mentioned = prompt.match_indices(&needle).any(|(i, _)| !prompt[i + needle.len()..].chars().next().is_some_and(|c| c.is_alphanumeric() || c == '_')); + if !mentioned { + continue; + } + let candidates: Vec<&crate::backend::wire::Character> = characters.iter().filter(|c| c.name == name).collect(); + let chosen = picks.get(name).and_then(|t| candidates.iter().find(|c| &c.token == t)).or(candidates.first()); + if let Some(c) = chosen { + tokens.push(c.token.clone()); + } + } + tokens +} + fn deck_limits(m: &ModelInfo, library: bool) -> DeckLimits { DeckLimits { max_images: m.image_refs_max, max_videos: m.video_refs_max, max_video_secs: m.video_refs_max_secs.map(f32::from), max_audios: m.audio_refs_max, max_audio_secs: m.audio_refs_max_secs.map(f32::from), library } } @@ -539,3 +615,25 @@ fn frames_full_message(model: Option<&ModelInfo>) -> &'static str { "The first frame is already set" } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::backend::wire::Character; + + fn character(token: &str, name: &str) -> Character { + Character { token: token.into(), name: name.into(), ..Default::default() } + } + + #[test] + fn mentions_resolve_to_one_token_per_name() { + // Newest first, as the server lists them. + let characters = [character("c3", "Bob"), character("c2", "Bob2"), character("c1", "Bob"), character("c0", "Ann")]; + let none = std::collections::HashMap::new(); + assert_eq!(mentioned_characters("@Bob2 waves", &characters, &none), ["c2"], "@Bob2 isn't @Bob"); + assert_eq!(mentioned_characters("@Bob, then @Ann.", &characters, &none), ["c3", "c0"], "newest Bob by default"); + let picks = std::collections::HashMap::from([("Bob".to_owned(), "c1".to_owned())]); + assert_eq!(mentioned_characters("hi @Bob", &characters, &picks), ["c1"], "the explicit pick wins"); + assert!(mentioned_characters("no mentions", &characters, &none).is_empty()); + } +} diff --git a/artcraft-native/src/pages/mod.rs b/artcraft-native/src/pages/mod.rs index a20581c603..e56ca7d922 100644 --- a/artcraft-native/src/pages/mod.rs +++ b/artcraft-native/src/pages/mod.rs @@ -1,4 +1,5 @@ pub mod common; +pub mod create_audio; pub mod create_image; pub mod create_video; pub mod other; diff --git a/artcraft-native/src/pages/other.rs b/artcraft-native/src/pages/other.rs index 106ae11f52..882794f17a 100644 --- a/artcraft-native/src/pages/other.rs +++ b/artcraft-native/src/pages/other.rs @@ -23,12 +23,13 @@ pub fn home(ui: &mut Ui, signed_in: bool, requests: &mut Vec) -> Opt ui.add_space(10.0); ui.label(widgets::heading("Capable tools for artists.", 52.0)); ui.add_space(10.0); - ui.label(egui::RichText::new("Generate images and videos with the best models, steer them with references and keyframes, and keep everything in your library.").size(17.0).color(theme::MUTED)); + ui.label(egui::RichText::new("Generate images, videos and audio with the best models, steer them with references and keyframes, and keep everything in your library.").size(17.0).color(theme::MUTED)); ui.add_space(28.0); ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 16.0; - let card_w = (width - 16.0) / 2.0; - for (page, icon, title, text) in [(Page::CreateImage, Icon::Image, "Create Image", "Describe anything. See it in seconds."), (Page::CreateVideo, Icon::Video, "Create Video", "Describe a scene. See it in motion.")] { + let cards = [(Page::CreateImage, Icon::Image, "Create Image", "Describe anything. See it in seconds."), (Page::CreateVideo, Icon::Video, "Create Video", "Describe a scene. See it in motion."), (Page::CreateAudio, Icon::Music, "Create Audio", "Describe a song, a sound, or a sample.")]; + let card_w = (width - 16.0 * (cards.len() - 1) as f32) / cards.len() as f32; + for (page, icon, title, text) in cards { if card(ui, card_w, icon, title, text) { open = Some(page); } @@ -73,9 +74,9 @@ pub fn library(ui: &mut Ui, env: &mut Env<'_>, feed: &mut FeedStore) { placeholder(ui, area, "Library", "Your generations will appear here.", None); return; } - let view = FeedView { id: Id::new("library-feed"), mode: env.view_mode, pending: &[], failed: &[], items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: true, bottom_padding: 24.0 }; + let view = FeedView { id: Id::new("library-feed"), mode: env.view_mode, pending: &[], failed: &[], items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: true, autoplay: env.autoplay, bottom_padding: 24.0 }; let mut child = ui.new_child(egui::UiBuilder::new().max_rect(area.shrink2(vec2(12.0, 0.0)).with_min_y(area.top() + 2.0))); - for action in grid::show(&mut child, &view, env.cache, env.ratios, env.catalog) { + for action in grid::show(&mut child, &view, env.cache, env.ratios, env.audio, env.catalog) { match action { FeedAction::Open(token) => { let kind = feed.find(&token).map_or(MediaKind::Image, |i| i.kind); diff --git a/artcraft-native/src/prompt_box/deck.rs b/artcraft-native/src/prompt_box/deck.rs index f33db02bee..185ea2cf84 100644 --- a/artcraft-native/src/prompt_box/deck.rs +++ b/artcraft-native/src/prompt_box/deck.rs @@ -309,12 +309,14 @@ fn paint_card(ui: &Ui, painter: &egui::Painter, rect: Rect, tilt: f32, item: &Re image.paint_at(ui, rect.shrink(2.0)); }, None => { + let playing = item.kind == RefKind::Audio && playing_audio(ui) == Some(item.id); let icon = match item.kind { RefKind::Image => Icon::Image, RefKind::Video => Icon::Video, - RefKind::Audio => Icon::Music, + RefKind::Audio if playing => Icon::Volume, + RefKind::Audio => Icon::Play, }; - icons::paint(painter, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), icon, theme::MUTED); + icons::paint(painter, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), icon, if playing { theme::ACCENT_INK } else { theme::MUTED }); }, } if item.kind != RefKind::Image && tilt == 0.0 { @@ -334,6 +336,17 @@ fn paint_card(ui: &Ui, painter: &egui::Painter, rect: Rect, tilt: f32, item: &Re painter.add(Shape::closed_line(corners.to_vec(), Stroke::new(2.0, border))); } +/// The audio reference playing right now (the app sets it each frame). +pub fn set_playing_audio(ctx: &egui::Context, id: Option) { + ctx.data_mut(|d| d.insert_temp(Id::new(PLAYING_AUDIO), id)); +} + +fn playing_audio(ui: &Ui) -> Option { + ui.data(|d| d.get_temp::>(Id::new(PLAYING_AUDIO))).flatten() +} + +const PLAYING_AUDIO: &str = "deck-playing-audio"; + /// The hover-revealed round × in a card's top-right corner. Returns whether it was clicked. fn remove_button(ui: &mut Ui, id: Id, card: Rect, card_hovered: bool) -> bool { let rect = Rect::from_min_size(pos2(card.right() - 22.0, card.top() + 2.0), vec2(20.0, 20.0)); diff --git a/artcraft-native/src/prompt_box/editor.rs b/artcraft-native/src/prompt_box/editor.rs index 42714c1b7b..307ad9d849 100644 --- a/artcraft-native/src/prompt_box/editor.rs +++ b/artcraft-native/src/prompt_box/editor.rs @@ -22,12 +22,15 @@ const RIGHT_GUTTER: f32 = 32.0; const IMAGE_COLORS: [Color32; 5] = [Color32::from_rgb(96, 165, 250), Color32::from_rgb(251, 146, 60), Color32::from_rgb(167, 139, 250), Color32::from_rgb(52, 211, 153), Color32::from_rgb(251, 113, 133)]; const VIDEO_COLORS: [Color32; 3] = [Color32::from_rgb(250, 204, 21), Color32::from_rgb(245, 158, 11), Color32::from_rgb(74, 222, 128)]; const AUDIO_COLORS: [Color32; 2] = [Color32::from_rgb(192, 132, 252), Color32::from_rgb(232, 121, 249)]; +/// Teal, emerald, sky. +const CHARACTER_COLORS: [Color32; 3] = [Color32::from_rgb(45, 212, 191), Color32::from_rgb(34, 197, 94), Color32::from_rgb(14, 165, 233)]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MentionKind { Image, Video, Audio, + Character, } /// Something the prompt can reference by name, e.g. `@Image2`. @@ -39,6 +42,8 @@ pub struct MentionItem { pub index: usize, /// Media cache key of a thumbnail, if there is one. pub preview: Option, + /// The character this mention names (several can share a name). + pub token: Option, } impl MentionItem { @@ -48,6 +53,7 @@ impl MentionItem { MentionKind::Image => IMAGE_COLORS[i % IMAGE_COLORS.len()], MentionKind::Video => VIDEO_COLORS[i % VIDEO_COLORS.len()], MentionKind::Audio => AUDIO_COLORS[i % AUDIO_COLORS.len()], + MentionKind::Character => CHARACTER_COLORS[i % CHARACTER_COLORS.len()], } } } @@ -93,6 +99,8 @@ pub struct EditorOutput { pub submit: bool, pub open_fullscreen: bool, pub focused: bool, + /// A mention the user picked from the dropdown: (label, character token). + pub picked: Option<(String, Option)>, } /// Draws the editor in the available width. @@ -105,7 +113,7 @@ pub fn show(ui: &mut Ui, prompt: &mut String, state: &mut EditorState, opts: &Ed // Keys the field mustn't see: dropdown navigation, and Enter when it generates. if had_focus { if !filtered.is_empty() { - handle_dropdown_keys(ui, id, prompt, state, &filtered); + out.picked = handle_dropdown_keys(ui, id, prompt, state, &filtered); } else if opts.enter_to_generate && consume_plain_enter(ui) { out.submit = true; } @@ -183,7 +191,9 @@ pub fn show(ui: &mut Ui, prompt: &mut String, state: &mut EditorState, opts: &Ed } if !filtered.is_empty() && out.focused { - mention_dropdown(ui, id, field_rect, prompt, state, &filtered, thumbnails); + if let Some(picked) = mention_dropdown(ui, id, field_rect, prompt, state, &filtered, thumbnails) { + out.picked = Some(picked); + } } out } @@ -214,12 +224,10 @@ fn consume_plain_enter(ui: &Ui) -> bool { }) } -fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem]) { +fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem]) -> Option<(String, Option)> { let n = filtered.len(); let (down, up, accept, escape) = ui.input_mut(|i| (i.consume_key(Modifiers::NONE, Key::ArrowDown), i.consume_key(Modifiers::NONE, Key::ArrowUp), i.consume_key(Modifiers::NONE, Key::Enter) || i.consume_key(Modifiers::NONE, Key::Tab), i.consume_key(Modifiers::NONE, Key::Escape))); - let Some(query) = state.mention.as_mut() else { - return; - }; + let query = state.mention.as_mut()?; if down { query.selected = (query.selected + 1) % n; } @@ -228,12 +236,14 @@ fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut Editor } if escape { state.mention = None; - return; + return None; } - if accept { - let label = filtered[query.selected.min(n - 1)].label.clone(); - insert_mention(ui.ctx(), id, prompt, state, &label); + if !accept { + return None; } + let item = filtered[query.selected.min(n - 1)]; + insert_mention(ui.ctx(), id, prompt, state, &item.label); + Some((item.label.clone(), item.token.clone())) } /// Opens, updates or closes the dropdown from the text before the cursor. @@ -284,7 +294,7 @@ fn insert_mention(ctx: &egui::Context, id: Id, prompt: &mut String, state: &mut } } -fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem], thumbnails: &mut dyn FnMut(&Ui, &str) -> Option) { +fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem], thumbnails: &mut dyn FnMut(&Ui, &str) -> Option) -> Option<(String, Option)> { let selected = state.mention.as_ref().map_or(0, |q| q.selected); let row_h = 44.0; let h = 28.0 + row_h * filtered.len() as f32; @@ -317,23 +327,25 @@ fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &m MentionKind::Image => Icon::Image, MentionKind::Video => Icon::Video, MentionKind::Audio => Icon::Music, + MentionKind::Character => Icon::User, }; icons::paint(ui.painter(), tile.shrink(9.0), icon, theme::fade(Color32::WHITE, 0.6)); }, } ui.painter().text(pos2(tile.right() + 10.0, row.center().y), Align2::LEFT_CENTER, &item.label, FontId::new(14.0, theme::medium()), item.color()); if resp.clicked() { - picked = Some(item.label.clone()); + picked = Some((item.label.clone(), item.token.clone())); } } }); if let (Some(i), Some(q)) = (hovered, state.mention.as_mut()) { q.selected = i; } - if let Some(label) = picked { - insert_mention(ui.ctx(), id, prompt, state, &label); + if let Some((label, _)) = &picked { + insert_mention(ui.ctx(), id, prompt, state, label); ui.memory_mut(|m| m.request_focus(id)); } + picked } /// The three-line grip that resizes the field (`.promptbox-resize-wrap::after`). @@ -382,14 +394,20 @@ fn highlight(text: &str, mentions: &[MentionItem]) -> LayoutJob { job } +/// `@Name` mentions for characters (newest first), coloured by position. +pub fn character_mentions<'a>(characters: impl Iterator)>) -> Vec { + characters.enumerate().map(|(i, (name, token, avatar))| MentionItem { label: format!("@{name}"), kind: MentionKind::Character, index: i + 1, preview: avatar, token: Some(token.to_owned()) }).collect() +} + /// Mention labels for `n` items of `kind` (`@Image1`, `@Image2`, …). pub fn mention_items(kind: MentionKind, previews: impl Iterator>) -> Vec { let name = match kind { MentionKind::Image => "Image", MentionKind::Video => "Video", MentionKind::Audio => "Audio", + MentionKind::Character => "Character", }; - previews.enumerate().map(|(i, preview)| MentionItem { label: format!("@{name}{}", i + 1), kind, index: i + 1, preview }).collect() + previews.enumerate().map(|(i, preview)| MentionItem { label: format!("@{name}{}", i + 1), kind, index: i + 1, preview, token: None }).collect() } #[cfg(test)] diff --git a/artcraft-native/src/prompt_box/mod.rs b/artcraft-native/src/prompt_box/mod.rs index 92e1afe38a..4390b47be6 100644 --- a/artcraft-native/src/prompt_box/mod.rs +++ b/artcraft-native/src/prompt_box/mod.rs @@ -58,6 +58,21 @@ pub struct PromptBoxProps<'a> { pub warning: Option<&'a str>, /// An info strip above the prompt. pub banner: Option<&'a str>, + /// Something clear-all also clears besides the prompt and references (the audio style). + pub extra_input: bool, +} + +/// Where a page draws into the box. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Slot { + /// Rows above the prompt (the audio box's references). + AbovePrompt, + /// Rows below the prompt (the audio box's style). + BelowPrompt, + /// The toolbar's left: the model selector and pickers. + ToolbarLeft, + /// The toolbar's right, before clear-all: the count picker. + ToolbarRight, } /// Something the page should do. @@ -68,6 +83,8 @@ pub enum PromptBoxAction { ClearAll, DroppedFiles(Vec), PastedImage(Vec), + /// A mention picked from the dropdown: (label, character token). + MentionPicked(String, Option), } /// UI state that outlives a frame. @@ -79,11 +96,10 @@ pub struct PromptBoxState { focused: bool, } -/// Draws the box in `ui` (which should be as wide as the box). `toolbar_left` holds the model -/// selector and pickers; `toolbar_right` the count picker. Both may be drawn twice in a frame -/// (box and focus mode), each under its own id. -#[allow(clippy::too_many_arguments)] -pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, toolbar_left: &mut dyn FnMut(&mut Ui), toolbar_right: &mut dyn FnMut(&mut Ui)) -> Vec { +/// Draws the box in `ui` (which should be as wide as the box). `slots` draws the page's parts +/// into each [`Slot`]; the toolbar's left may be drawn twice in a frame (box and focus mode), each +/// under its own id. +pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, slots: &mut dyn FnMut(&mut Ui, Slot)) -> Vec { let mut actions = Vec::new(); let frame = egui::Frame::new().fill(theme::fade(theme::CONTROLS, 0.95)).stroke(Stroke::new(1.0, if state.focused { theme::ACCENT } else { theme::LINE })).corner_radius(theme::RADIUS).inner_margin(16); let response = frame.show(ui, |ui| { @@ -94,25 +110,30 @@ pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut Pro ui.spacing_mut().item_spacing.x = 12.0; deck_widget(ui, props, refs, cache, false, &mut actions); ui.vertical(|ui| { + slots(ui, Slot::AbovePrompt); let out = editor_ui(ui, prompt, state, props, cache, None); state.focused = out.focused; if out.submit { actions.push(PromptBoxAction::Generate); } + if let Some((label, token)) = out.picked { + actions.push(PromptBoxAction::MentionPicked(label, token)); + } if out.open_fullscreen { state.fullscreen = true; } + slots(ui, Slot::BelowPrompt); }); }); ui.add_space(14.0); ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 8.0; - toolbar_left(ui); + slots(ui, Slot::ToolbarLeft); ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { ui.spacing_mut().item_spacing.x = 6.0; generate_cluster(ui, props, &mut actions); - clear_all_button(ui, prompt, refs, state, &mut actions); - toolbar_right(ui); + clear_all_button(ui, prompt, refs, state, props.extra_input, &mut actions); + slots(ui, Slot::ToolbarRight); if let Some(warning) = props.warning { warning_label(ui, warning); } @@ -124,7 +145,7 @@ pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut Pro drop_target(ui, props, rect, &mut actions); paste_images(ui, props, &mut actions); if state.fullscreen { - focus_mode(ui.ctx(), prompt, refs, state, props, cache, toolbar_left, &mut actions); + focus_mode(ui.ctx(), prompt, refs, state, props, cache, slots, &mut actions); } if state.confirm_clear { confirm_clear(ui.ctx(), props.id, state, &mut actions); @@ -167,10 +188,10 @@ fn generate_cluster(ui: &mut Ui, props: &PromptBoxProps<'_>, actions: &mut Vec

) { +fn clear_all_button(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, extra_input: bool, actions: &mut Vec) { let (rule, _) = ui.allocate_exact_size(vec2(1.0, 20.0), Sense::hover()); ui.painter().rect_filled(rule, 0.0, theme::fade(theme::INK, 0.15)); - let clearable = !prompt.is_empty() || !refs.is_empty(); + let clearable = !prompt.is_empty() || !refs.is_empty() || extra_input; let resp = ui.add_enabled_ui(clearable, |ui| widgets::ghost(ui, Icon::Eraser, None, "Clear all", false)).inner; if resp.clicked() { if refs.is_empty() { @@ -307,7 +328,7 @@ fn paste_images(ui: &mut Ui, props: &PromptBoxProps<'_>, actions: &mut Vec, cache: &mut MediaCache, toolbar_left: &mut dyn FnMut(&mut Ui), actions: &mut Vec) { +fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, slots: &mut dyn FnMut(&mut Ui, Slot), actions: &mut Vec) { let height = ctx.content_rect().height() * 0.7; let mut done = false; let closed = widgets::modal(ctx, props.id.with("focus-mode"), 896.0, |ui| { @@ -330,12 +351,12 @@ fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state ui.push_id("focus-deck", |ui| deck_widget(ui, props, refs, cache, true, actions)); ui.add_space(12.0); ui.horizontal(|ui| { - ui.push_id("focus-toolbar", |ui| ui.horizontal(|ui| toolbar_left(ui))); + ui.push_id("focus-toolbar", |ui| ui.horizontal(|ui| slots(ui, Slot::ToolbarLeft))); ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { if widgets::button(ui, None, "Done", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() { done = true; } - ui.push_id("focus-clear", |ui| clear_all_button(ui, prompt, refs, state, actions)); + ui.push_id("focus-clear", |ui| clear_all_button(ui, prompt, refs, state, props.extra_input, actions)); }); }); }); diff --git a/artcraft-native/src/prompt_box/pickers.rs b/artcraft-native/src/prompt_box/pickers.rs index 5b09b7581a..0530222580 100644 --- a/artcraft-native/src/prompt_box/pickers.rs +++ b/artcraft-native/src/prompt_box/pickers.rs @@ -90,14 +90,19 @@ pub fn popover(anchor: &Response, popup_id: Id, content: impl FnOnce(&mut Ui)) { }); } +/// A toolbar pill that opens a popover of custom `content` above it. +pub fn popover_pill(ui: &mut Ui, id: Id, lead: Leading<'_>, label: &str, tooltip: &str, content: impl FnOnce(&mut Ui)) { + let popup_id = ui.make_persistent_id(id).with("popup"); + let open = Popup::is_id_open(ui.ctx(), popup_id); + let resp = ui.push_id(id, |ui| widgets::pill(ui, lead, label, open)).inner.on_hover_text(tooltip); + popover(&resp, popup_id, content); +} + /// The duration popover: a clock pill reading `5s`, opening a 1-second-step slider between the /// model's minimum and maximum. Returns the new value. pub fn duration_picker(ui: &mut Ui, id: Id, current: u32, min: u32, max: u32) -> Option { - let popup_id = ui.make_persistent_id(id).with("popup"); - let open = Popup::is_id_open(ui.ctx(), popup_id); - let resp = ui.push_id(id, |ui| widgets::pill(ui, Leading::Icon(Icon::Clock), &format!("{current}s"), open)).inner.on_hover_text("Duration"); let mut value = current; - popover(&resp, popup_id, |ui| { + popover_pill(ui, id, Leading::Icon(Icon::Clock), &format!("{current}s"), "Duration", |ui| { ui.set_width(240.0); widgets::menu_header(ui, "Duration"); ui.add_space(4.0); diff --git a/artcraft-native/src/prompt_box/ui_tests.rs b/artcraft-native/src/prompt_box/ui_tests.rs index 7baa196ad8..c752bf40cc 100644 --- a/artcraft-native/src/prompt_box/ui_tests.rs +++ b/artcraft-native/src/prompt_box/ui_tests.rs @@ -11,7 +11,7 @@ use crate::pages::toolbar; use crate::prompt_box::deck::DeckLimits; use crate::prompt_box::editor::{MentionKind, mention_items}; use crate::prompt_box::refs::{RefKind, References}; -use crate::prompt_box::{DeckMode, PromptBoxProps, PromptBoxState, show}; +use crate::prompt_box::{DeckMode, PromptBoxProps, PromptBoxState, Slot, show}; struct State { // Keeps the media cache's runtime alive. @@ -45,15 +45,16 @@ fn build(state: State) -> Harness<'static, State> { return; } let mentions = if s.with_mentions { mention_items(MentionKind::Image, [None, None].into_iter()) } else { Vec::new() }; - let props = PromptBoxProps { id: egui::Id::new("test-box"), placeholder: "Describe what you want in the image...", enter_to_generate: s.enter_to_generate, mentions: &mentions, max_length: Some(Some(1000)), deck: DeckMode::References(DeckLimits { max_images: 4, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: false }), accepts: &[RefKind::Image], credits: Some(15), generate_enabled: !s.prompt.trim().is_empty(), generating: false, generate_tooltip: "Generate", warning: None, banner: None }; + let props = PromptBoxProps { id: egui::Id::new("test-box"), placeholder: "Describe what you want in the image...", enter_to_generate: s.enter_to_generate, mentions: &mentions, max_length: Some(Some(1000)), deck: DeckMode::References(DeckLimits { max_images: 4, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: false }), accepts: &[RefKind::Image], credits: Some(15), generate_enabled: !s.prompt.trim().is_empty(), generating: false, generate_tooltip: "Generate", warning: None, banner: None, extra_input: false }; let (model, aspect) = (&s.model, &mut s.aspect); - let mut left = |ui: &mut egui::Ui| { - if let Some(v) = toolbar::aspect_ratio(ui, egui::Id::new("test-aspect"), model, aspect) { - *aspect = v; + let mut slots = |ui: &mut egui::Ui, slot: Slot| { + if slot == Slot::ToolbarLeft { + if let Some(v) = toolbar::aspect_ratio(ui, egui::Id::new("test-aspect"), model, aspect) { + *aspect = v; + } } }; - let mut right = |_: &mut egui::Ui| {}; - let actions = show(ui, &mut s.prompt, &s.refs, &mut s.box_state, &props, &mut s.cache, &mut left, &mut right); + let actions = show(ui, &mut s.prompt, &s.refs, &mut s.box_state, &props, &mut s.cache, &mut slots); s.actions.extend(actions.iter().map(|a| format!("{a:?}"))); }, state, @@ -128,7 +129,7 @@ fn typing_at_offers_mentions_and_enter_inserts_one() { harness.key_press(egui::Key::Enter); harness.run(); assert_eq!(harness.state().prompt, "put @Image2 "); - assert!(harness.state().actions.is_empty(), "Enter picked a mention, it didn't generate"); + assert_eq!(harness.state().actions, [r#"MentionPicked("@Image2", None)"#], "Enter picked a mention, it didn't generate"); } #[test] diff --git a/artcraft-native/src/shell.rs b/artcraft-native/src/shell.rs index ebeb4e0df9..110a7eddb3 100644 --- a/artcraft-native/src/shell.rs +++ b/artcraft-native/src/shell.rs @@ -87,6 +87,7 @@ impl Page { "home" => Page::Home, "create-image" | "image" => Page::CreateImage, "create-video" | "video" => Page::CreateVideo, + "create-audio" | "audio" => Page::CreateAudio, "library" => Page::Library, _ => return None, }) @@ -94,7 +95,7 @@ impl Page { /// Pages the native app has; the rest open a "coming soon" page. pub fn is_ready(self) -> bool { - matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::Library) + matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::CreateAudio | Page::Library) } /// The same tool in the web app. @@ -133,6 +134,7 @@ pub enum ShellAction { SignOut, OpenSettings, SetViewMode(ViewMode), + SetAutoplay(bool), ToggleSelect, } @@ -286,8 +288,17 @@ fn account_row(ui: &mut Ui, name: &str, credits: Option) -> egui::Response resp.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_text("Settings") } -/// The content panel's header: breadcrumbs, and the select and grid/list toggles on feed pages. -pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option<(ViewMode, bool)>) -> Option { +/// The feed toggles a page's header shows. +pub struct FeedToggles { + pub mode: ViewMode, + pub selecting: bool, + /// Shown on pages with videos (`GalleryAutoplayToggle`). + pub autoplay: Option, +} + +/// The content panel's header: breadcrumbs, and the select, previews and grid/list toggles on +/// feed pages. +pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option) -> Option { let mut action = None; let rect = ui.max_rect(); ui.painter().hline(rect.x_range(), rect.bottom() - 0.5, theme::hairline()); @@ -302,12 +313,18 @@ pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option<(ViewMode, bool let (crumb, _) = ui.allocate_exact_size(galley.size() + vec2(12.0, 8.0), Sense::hover()); ui.painter().rect_filled(crumb, 0.0, theme::INK); ui.painter().galley(crumb.center() - galley.size() / 2.0, galley, theme::BG); - if let Some((mode, selecting)) = feed_toggles { + if let Some(t) = feed_toggles { ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - if let Some(m) = view_toggle(ui, mode) { + if let Some(m) = view_toggle(ui, t.mode) { action = Some(ShellAction::SetViewMode(m)); } - if segmented(ui, &[(Icon::SquareCheck, if selecting { "Exit selection" } else { "Select items" }, selecting)]).is_some() { + if let Some(playing) = t.autoplay { + // Right-to-left: stills first so "play" ends up on the left. + if let Some(i) = segmented(ui, &[(Icon::Image, "Still thumbnails", !playing), (Icon::Play, "Play video previews", playing)]) { + action = Some(ShellAction::SetAutoplay(i == 1)); + } + } + if segmented(ui, &[(Icon::SquareCheck, if t.selecting { "Exit selection" } else { "Select items" }, t.selecting)]).is_some() { action = Some(ShellAction::ToggleSelect); } }); diff --git a/artcraft-native/src/ui/icons.rs b/artcraft-native/src/ui/icons.rs index 7467aafd4b..51089ad06f 100644 --- a/artcraft-native/src/ui/icons.rs +++ b/artcraft-native/src/ui/icons.rs @@ -18,6 +18,7 @@ pub enum Icon { Coins, Copy, Download, + Drum, Eraser, Eye, ExternalLink, @@ -35,17 +36,24 @@ pub enum Icon { LogIn, LogOut, Maximize, + Mic, + MicOff, Music, PanelLeft, + Pause, Pencil, Play, Plus, + Repeat, RotateCw, Settings, + SlidersHorizontal, Sparkles, SquareCheck, + Tags, Trash, Upload, + User, Video, Volume, VolumeOff, @@ -157,6 +165,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { g.line(p, &[(7.0, 10.0), (12.0, 15.0), (17.0, 10.0)]); g.line(p, &[(4.0, 21.0), (20.0, 21.0)]); }, + Icon::Drum => { + g.line(p, &[(2.0, 2.0), (10.0, 8.0)]); + g.line(p, &[(22.0, 2.0), (14.0, 8.0)]); + g.closed(p, &[(2.0, 9.0), (6.0, 6.5), (12.0, 5.5), (18.0, 6.5), (22.0, 9.0), (18.0, 11.5), (12.0, 12.5), (6.0, 11.5)]); + g.line(p, &[(2.0, 9.0), (2.0, 17.0), (6.0, 19.5), (12.0, 20.5), (18.0, 19.5), (22.0, 17.0), (22.0, 9.0)]); + g.line(p, &[(7.0, 11.8), (7.0, 19.8)]); + g.line(p, &[(17.0, 11.8), (17.0, 19.8)]); + }, Icon::Eraser => { g.closed(p, &[(7.0, 21.0), (3.5, 17.5), (14.0, 7.0), (21.0, 14.0), (14.0, 21.0)]); g.line(p, &[(22.0, 21.0), (7.0, 21.0)]); @@ -248,6 +264,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { g.line(p, &[(21.0, 3.0), (14.0, 10.0)]); g.line(p, &[(3.0, 21.0), (10.0, 14.0)]); }, + Icon::Mic | Icon::MicOff => { + g.rect(p, 9.0, 2.0, 6.0, 13.0, 3.0); + g.line(p, &[(19.0, 10.0), (19.0, 12.0), (17.0, 16.0), (12.0, 19.0), (7.0, 16.0), (5.0, 12.0), (5.0, 10.0)]); + g.line(p, &[(12.0, 19.0), (12.0, 22.0)]); + if icon == Icon::MicOff { + g.line(p, &[(2.0, 2.0), (22.0, 22.0)]); + } + }, Icon::Music => { g.line(p, &[(9.0, 18.0), (9.0, 5.0), (21.0, 3.0), (21.0, 16.0)]); g.circle(p, 6.0, 18.0, 3.0); @@ -257,6 +281,10 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { g.rect(p, 3.0, 3.0, 18.0, 18.0, 2.0); g.line(p, &[(9.0, 3.0), (9.0, 21.0)]); }, + Icon::Pause => { + p.rect_filled(Rect::from_min_size(g.p(6.0, 4.0), vec2(4.0, 16.0) * g.scale), 1.0, color); + p.rect_filled(Rect::from_min_size(g.p(14.0, 4.0), vec2(4.0, 16.0) * g.scale), 1.0, color); + }, Icon::Pencil => { g.closed(p, &[(17.0, 3.0), (21.0, 7.0), (7.5, 20.5), (2.0, 22.0), (3.5, 16.5)]); g.line(p, &[(15.0, 5.0), (19.0, 9.0)]); @@ -269,6 +297,12 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { g.line(p, &[(5.0, 12.0), (19.0, 12.0)]); g.line(p, &[(12.0, 5.0), (12.0, 19.0)]); }, + Icon::Repeat => { + g.line(p, &[(17.0, 2.0), (21.0, 6.0), (17.0, 10.0)]); + g.line(p, &[(3.0, 11.0), (3.0, 10.0), (5.0, 7.0), (7.0, 6.0), (21.0, 6.0)]); + g.line(p, &[(7.0, 22.0), (3.0, 18.0), (7.0, 14.0)]); + g.line(p, &[(21.0, 13.0), (21.0, 14.0), (19.0, 17.0), (17.0, 18.0), (3.0, 18.0)]); + }, Icon::RotateCw => { g.arc(p, 12.0, 12.0, 9.0, -60.0, 250.0); g.line(p, &[(21.0, 3.0), (21.0, 8.5), (15.5, 8.5)]); @@ -281,6 +315,17 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { g.line(p, &[(12.0 + 7.5 * a.cos(), 12.0 + 7.5 * a.sin()), (12.0 + 10.0 * a.cos(), 12.0 + 10.0 * a.sin())]); } }, + Icon::SlidersHorizontal => { + g.line(p, &[(21.0, 4.0), (14.0, 4.0)]); + g.line(p, &[(10.0, 4.0), (3.0, 4.0)]); + g.line(p, &[(21.0, 12.0), (12.0, 12.0)]); + g.line(p, &[(8.0, 12.0), (3.0, 12.0)]); + g.line(p, &[(21.0, 20.0), (16.0, 20.0)]); + g.line(p, &[(12.0, 20.0), (3.0, 20.0)]); + g.line(p, &[(14.0, 2.0), (14.0, 6.0)]); + g.line(p, &[(8.0, 10.0), (8.0, 14.0)]); + g.line(p, &[(16.0, 18.0), (16.0, 22.0)]); + }, Icon::Sparkles => { let star = |cx: f32, cy: f32, r: f32| { let k = r * 0.28; @@ -289,6 +334,11 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { star(10.0, 12.0, 8.0); star(19.0, 5.0, 3.0); }, + Icon::Tags => { + g.closed(p, &[(3.0, 5.0), (5.0, 3.0), (10.6, 3.0), (18.0, 10.4), (18.0, 12.6), (12.6, 18.0), (10.4, 18.0), (3.0, 10.6)]); + g.line(p, &[(15.0, 3.0), (22.0, 10.0), (22.0, 12.0), (17.0, 17.0)]); + g.dot(p, 7.5, 7.5, 1.1); + }, Icon::Trash => { g.line(p, &[(3.0, 6.0), (21.0, 6.0)]); g.line(p, &[(19.0, 6.0), (19.0, 20.0), (17.0, 22.0), (7.0, 22.0), (5.0, 20.0), (5.0, 6.0)]); @@ -299,6 +349,10 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { g.line(p, &[(7.0, 8.0), (12.0, 3.0), (17.0, 8.0)]); g.line(p, &[(4.0, 21.0), (20.0, 21.0)]); }, + Icon::User => { + g.circle(p, 12.0, 8.0, 4.5); + g.arc(p, 12.0, 22.0, 8.0, 180.0, 360.0); + }, Icon::Video => { g.rect(p, 2.0, 6.0, 14.0, 12.0, 2.0); g.closed(p, &[(16.0, 10.5), (22.0, 7.0), (22.0, 17.0), (16.0, 13.5)]); diff --git a/artcraft-native/src/ui/mod.rs b/artcraft-native/src/ui/mod.rs index 36c2450504..b83d65f9b9 100644 --- a/artcraft-native/src/ui/mod.rs +++ b/artcraft-native/src/ui/mod.rs @@ -1,5 +1,6 @@ pub mod creator_icons; pub mod icons; pub mod toast; +pub mod waveform; pub mod widgets; pub mod window; diff --git a/artcraft-native/src/ui/waveform.rs b/artcraft-native/src/ui/waveform.rs new file mode 100644 index 0000000000..4b66929525 --- /dev/null +++ b/artcraft-native/src/ui/waveform.rs @@ -0,0 +1,194 @@ +//! The track player (`WaveformAudioPlayer`): a round white play button, the waveform (click to +//! seek, played part in blue), the time, and a volume flyout. Placeholder bars stand in while the +//! waveform decodes; a plain progress line if it can't. + +use egui::{Color32, FontId, Id, Popup, Rect, Sense, Stroke, Ui, pos2, vec2}; + +use crate::backend::audio::{self, AudioPlayer, WaveformLookup}; +use crate::prompt_box::pickers; +use crate::theme; +use crate::ui::icons::{self, Icon}; + +/// Bars drawn while the waveform loads (`PLACEHOLDER_BAR_COUNT`). +const PLACEHOLDER_BARS: usize = 48; +/// Waveform bars: width and the step between them. +const BAR_WIDTH: f32 = 2.0; +const BAR_STEP: f32 = 3.5; +const WAVE_INK: Color32 = Color32::from_rgba_premultiplied(89, 89, 89, 89); +const PROGRESS_INK: Color32 = Color32::from_rgb(0x2d, 0x81, 0xff); +/// How often a playing track redraws its progress. +const FRAME: std::time::Duration = std::time::Duration::from_millis(33); + +/// Draws the player across `ui`'s width; `duration_hint` (seconds) is used until the waveform +/// knows the real length. `compact` is the list rows' smaller size. Returns the player's rect. +pub fn player(ui: &mut Ui, id: Id, audio: &mut AudioPlayer, url: &str, duration_hint: Option, compact: bool) -> Rect { + let (button_d, wave_h): (f32, f32) = if compact { (32.0, 32.0) } else { (40.0, 44.0) }; + let (rect, _) = ui.allocate_exact_size(vec2(ui.available_width(), wave_h.max(button_d)), Sense::hover()); + // Clicks between the controls stay with the player (they don't open the card underneath). + ui.interact(rect, id.with("bg"), Sense::click()); + let ctx = ui.ctx().clone(); + let waveform = audio.waveform(&ctx, url); + let playback = audio.track(url); + let duration = match &waveform { + WaveformLookup::Ready(w) => Some(w.duration), + _ => duration_hint, + } + .filter(|d| *d > 0.0); + let position = playback.map_or(0.0, |p| p.position); + let playing = playback.is_some_and(|p| !p.paused && !p.loading); + if playing { + ctx.request_repaint_after(FRAME); + } + + // Play / pause. + let button = Rect::from_center_size(pos2(rect.left() + button_d / 2.0, rect.center().y), vec2(button_d, button_d)); + let resp = ui.interact(button, id.with("play"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand); + let label = if playing { "Pause" } else { "Play" }; + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label)); + ui.painter().circle_filled(button.center(), button_d / 2.0, if resp.hovered() { theme::fade(Color32::WHITE, 0.88) } else { Color32::WHITE }); + let glyph = Rect::from_center_size(button.center(), vec2(button_d * 0.4, button_d * 0.4)); + match playback { + Some(p) if p.loading => icons::paint_spinner(ui, glyph, Color32::BLACK), + _ if playing => icons::paint(ui.painter(), glyph, Icon::Pause, Color32::BLACK), + _ => icons::paint(ui.painter(), glyph.translate(vec2(button_d * 0.04, 0.0)), Icon::Play, Color32::BLACK), + } + if resp.clicked() { + audio.toggle_track(&ctx, url); + } + + // Volume, then the time, from the right. + let volume_rect = Rect::from_center_size(pos2(rect.right() - 14.0, rect.center().y), vec2(28.0, 28.0)); + volume(ui, id, volume_rect, audio); + let time = format!("{} / {}", format_time(position), duration.map_or_else(|| "-:--".to_owned(), format_time)); + let galley = ui.painter().layout_no_wrap(time, FontId::new(11.0, theme::mono()), theme::MUTED); + let time_x = volume_rect.left() - 8.0 - galley.size().x; + ui.painter().galley(pos2(time_x, rect.center().y - galley.size().y / 2.0), galley, theme::MUTED); + + // The waveform between them. + let wave = Rect::from_min_max(pos2(button.right() + 10.0, rect.center().y - wave_h / 2.0), pos2(time_x - 10.0, rect.center().y + wave_h / 2.0)); + if wave.width() > 8.0 { + let progress = duration.map_or(0.0, |d| (position / d).clamp(0.0, 1.0)); + let wave_resp = ui.interact(wave, id.with("wave"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand); + match &waveform { + WaveformLookup::Ready(w) => paint_bars(ui, wave, &audio::resample(&w.peaks, bar_count(wave.width())), progress), + WaveformLookup::Loading => paint_bars(ui, wave, &placeholder_bars(url), progress), + WaveformLookup::Failed => paint_line(ui, wave, progress), + } + if let Some(pointer) = wave_resp.interact_pointer_pos().filter(|_| wave_resp.clicked()) { + let frac = ((pointer.x - wave.left()) / wave.width()).clamp(0.0, 1.0); + audio.seek_track(&ctx, url, duration.map_or(0.0, |d| frac * d)); + } + } + rect +} + +/// "m:ss". +pub fn format_time(secs: f32) -> String { + let s = secs.max(0.0) as u64; + format!("{}:{:02}", s / 60, s % 60) +} + +/// The volume button and its 0–100 slider flyout. +fn volume(ui: &mut Ui, id: Id, rect: Rect, audio: &mut AudioPlayer) { + let popup_id = id.with("volume-popup"); + let open = Popup::is_id_open(ui.ctx(), popup_id); + let resp = ui.interact(rect, id.with("volume"), Sense::click()).on_hover_text("Volume").on_hover_cursor(egui::CursorIcon::PointingHand); + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, "Volume")); + if resp.hovered() || open { + ui.painter().rect_filled(rect, theme::RADIUS, theme::WASH); + } + let level = audio.track_volume(); + icons::paint(ui.painter(), rect.shrink(7.0), if level <= 0.0 { Icon::VolumeOff } else { Icon::Volume }, if resp.hovered() || open { theme::INK } else { theme::DIM }); + let mut percent = (level * 100.0).round() as u32; + pickers::popover(&resp, popup_id, |ui| { + ui.horizontal(|ui| { + ui.add_space(6.0); + ui.style_mut().spacing.slider_width = 120.0; + ui.add(egui::Slider::new(&mut percent, 0..=100).show_value(false)); + ui.label(egui::RichText::new(percent.to_string()).size(11.0).family(theme::mono()).color(theme::MUTED)); + ui.add_space(4.0); + }); + }); + if percent != (level * 100.0).round() as u32 { + audio.set_track_volume(percent as f32 / 100.0); + } +} + +fn bar_count(width: f32) -> usize { + ((width + BAR_STEP - BAR_WIDTH) / BAR_STEP).floor().max(1.0) as usize +} + +/// Bars centred on the waveform's midline, blue up to `progress` (0–1). +fn paint_bars(ui: &Ui, rect: Rect, heights: &[f32], progress: f32) { + if heights.is_empty() { + return; + } + let step = rect.width() / heights.len() as f32; + let width = BAR_WIDTH.min(step * 0.7); + let split = rect.left() + rect.width() * progress; + let p = ui.painter(); + for (i, h) in heights.iter().enumerate() { + let x = rect.left() + step * (i as f32 + 0.5); + let half = (h.clamp(0.0, 1.0) * rect.height() / 2.0).max(1.0); + let bar = Rect::from_center_size(pos2(x, rect.center().y), vec2(width, half * 2.0)); + p.rect_filled(bar, 1.0, if x <= split { PROGRESS_INK } else { WAVE_INK }); + } +} + +/// The fallback when the waveform can't be decoded: a thin track with the played part in blue. +fn paint_line(ui: &Ui, rect: Rect, progress: f32) { + let track = Rect::from_center_size(rect.center(), vec2(rect.width(), 4.0)); + ui.painter().rect_filled(track, 2.0, WAVE_INK); + ui.painter().rect_filled(Rect::from_min_size(track.min, vec2(track.width() * progress, track.height())), 2.0, PROGRESS_INK); + ui.painter().line_segment([pos2(track.left() + track.width() * progress, rect.top() + 6.0), pos2(track.left() + track.width() * progress, rect.bottom() - 6.0)], Stroke::new(1.0, PROGRESS_INK)); +} + +/// Deterministic stand-in bars for a track (seeded by its URL), 25–89 % tall. +pub fn placeholder_bars(url: &str) -> Vec { + let mut seed = url.chars().fold(0_u32, |s, c| s.wrapping_mul(31).wrapping_add(c as u32)); + (0..PLACEHOLDER_BARS) + .map(|_| { + let percent = 25 + seed % 65; + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + percent as f32 / 100.0 + }) + .collect() +} + +/// A tile with a music note (`MusicIcon` in its box), for audio without artwork. +pub fn music_tile(ui: &Ui, rect: Rect, round: bool) { + let p = ui.painter(); + if round { + p.circle_filled(rect.center(), rect.width() / 2.0, theme::WASH_HOVER); + } else { + p.rect(rect, theme::RADIUS, theme::WASH, theme::hairline(), egui::StrokeKind::Inside); + } + icons::paint(p, Rect::from_center_size(rect.center(), rect.size() * 0.42), Icon::Music, theme::fade(Color32::WHITE, 0.75)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn times_read_minutes_and_seconds() { + assert_eq!(format_time(0.0), "0:00"); + assert_eq!(format_time(65.9), "1:05"); + assert_eq!(format_time(600.0), "10:00"); + } + + #[test] + fn placeholder_bars_are_stable_per_track() { + let a = placeholder_bars("https://cdn/a.mp3"); + assert_eq!(a.len(), PLACEHOLDER_BARS); + assert_eq!(a, placeholder_bars("https://cdn/a.mp3")); + assert_ne!(a, placeholder_bars("https://cdn/b.mp3")); + assert!(a.iter().all(|h| (0.25..0.9).contains(h))); + } + + #[test] + fn bars_fill_the_width() { + assert_eq!(bar_count(2.0), 1); + assert_eq!(bar_count(352.0), 101); + } +}