Compare commits

...
Author SHA1 Message Date
BFlat 28a39a0de4 artcraft-native: Create Audio, in-app playback, animated previews and @Characters (#2079)
Create Audio (PromptBoxAudio) with waveform playback, in-app video playback, animated feed previews, and @Character mentions with the Characters dialog.
2026-10-11 02:03:53 +07:00
BFlatandClaude Opus 5.5 bb27304b8b artcraft-native: custom title bar and Claude-style layout (#2068)
## What

The native app now draws its own window chrome, laid out like the Claude
desktop app.

- **Custom title bar** (40 px) replaces the native frame:
  - Left: the ArtCraft mark (goes Home) and the sidebar toggle.
- Right: credits, the account menu or Sign in, then minimize,
maximize/restore and close.
  - Dragging the empty space moves the window; a double-click maximizes.
  - On macOS the traffic lights stay, with room left for them.
- **Sidebar** is flush with the window's left edge, on the darker
chrome, with no floating card.
- **Content panel** is inset 8 px from the window edges, with rounded
corners and a hairline border. Its header holds the breadcrumbs and the
select and grid/list toggles.
- **Borderless window**: the edges and corners resize it, and a hairline
outlines it (hidden when maximized).

## Notes

- The window chrome helpers (`ui/window.rs`) are ported from the
ArtCraft Launcher, which uses the same egui 0.36.
- The mark is the in-repo `services/artcraft.svg`, tinted brand blue.
The old wordmark loader is removed.
- Toasts are moved down to sit below the new header.

## Testing

- `cargo test -p artcraft_native`: 29 tests pass.
- `cargo clippy -p artcraft_native --all-targets --no-deps` is clean.
- Screenshot-verified against a live account.
- Not tested by hand: dragging, resizing and the caption buttons.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 00:54:50 +07:00
36 changed files with 4183 additions and 471 deletions
Generated
+324 -7
View File
@@ -184,6 +184,28 @@ version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
[[package]]
name = "alsa"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43"
dependencies = [
"alsa-sys",
"bitflags 2.13.2",
"cfg-if",
"libc",
]
[[package]]
name = "alsa-sys"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527"
dependencies = [
"libc",
"pkg-config",
]
[[package]]
name = "android-activity"
version = "0.6.1"
@@ -196,9 +218,9 @@ dependencies = [
"jni 0.22.4",
"libc",
"log",
"ndk",
"ndk 0.9.0",
"ndk-context",
"ndk-sys",
"ndk-sys 0.6.0+11769913",
"num_enum",
"thiserror 2.0.18",
]
@@ -480,6 +502,7 @@ dependencies = [
"reqwest 0.12.23",
"resvg",
"rfd",
"rodio",
"serde",
"serde_json",
"tokens",
@@ -1730,6 +1753,49 @@ dependencies = [
"libm",
]
[[package]]
name = "coreaudio-rs"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace"
dependencies = [
"bitflags 1.3.2",
"core-foundation-sys",
"coreaudio-sys",
]
[[package]]
name = "coreaudio-sys"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953"
dependencies = [
"bindgen",
]
[[package]]
name = "cpal"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779"
dependencies = [
"alsa",
"core-foundation-sys",
"coreaudio-rs",
"dasp_sample",
"jni 0.21.1",
"js-sys",
"libc",
"mach2",
"ndk 0.8.0",
"ndk-context",
"oboe",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"windows 0.54.0",
]
[[package]]
name = "cpufeatures"
version = "0.2.12"
@@ -1930,6 +1996,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "dasp_sample"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
[[package]]
name = "data-encoding"
version = "2.5.0"
@@ -2696,6 +2768,12 @@ dependencies = [
"zune-inflate",
]
[[package]]
name = "extended"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fal_client"
version = "0.0.1"
@@ -4832,6 +4910,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mach2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
dependencies = [
"libc",
]
[[package]]
name = "markup5ever"
version = "0.35.0"
@@ -5075,6 +5162,20 @@ dependencies = [
"tempfile",
]
[[package]]
name = "ndk"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7"
dependencies = [
"bitflags 2.13.2",
"jni-sys 0.3.1",
"log",
"ndk-sys 0.5.0+25.2.9519653",
"num_enum",
"thiserror 1.0.63",
]
[[package]]
name = "ndk"
version = "0.9.0"
@@ -5084,7 +5185,7 @@ dependencies = [
"bitflags 2.13.2",
"jni-sys 0.3.1",
"log",
"ndk-sys",
"ndk-sys 0.6.0+11769913",
"num_enum",
"raw-window-handle",
"thiserror 1.0.63",
@@ -5096,6 +5197,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
[[package]]
name = "ndk-sys"
version = "0.5.0+25.2.9519653"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691"
dependencies = [
"jni-sys 0.3.1",
]
[[package]]
name = "ndk-sys"
version = "0.6.0+11769913"
@@ -5642,6 +5752,29 @@ dependencies = [
"objc2-foundation 0.3.2",
]
[[package]]
name = "oboe"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"
dependencies = [
"jni 0.21.1",
"ndk 0.8.0",
"ndk-context",
"num-derive",
"num-traits",
"oboe-sys",
]
[[package]]
name = "oboe-sys"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d"
dependencies = [
"cc",
]
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -6917,6 +7050,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "rodio"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7ceb6607dd738c99bc8cb28eff249b7cd5c8ec88b9db96c0608c1480d140fb1"
dependencies = [
"cpal",
"symphonia",
]
[[package]]
name = "ron"
version = "0.12.2"
@@ -8258,6 +8401,151 @@ dependencies = [
"serde_json",
]
[[package]]
name = "symphonia"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-adpcm",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-isomp4",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-adpcm"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73"
dependencies = [
"log",
"symphonia-core",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-core"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af"
dependencies = [
"arrayvec",
"bitflags 1.3.2",
"bytemuck",
"lazy_static",
"log",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5"
dependencies = [
"encoding_rs",
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-riff"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f"
dependencies = [
"extended",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16"
dependencies = [
"encoding_rs",
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-utils-xiph"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16"
dependencies = [
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "1.0.109"
@@ -8366,8 +8654,8 @@ dependencies = [
"jni 0.21.1",
"libc",
"log",
"ndk",
"ndk-sys",
"ndk 0.9.0",
"ndk-sys 0.6.0+11769913",
"objc2 0.6.4",
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
@@ -10447,6 +10735,16 @@ dependencies = [
"windows-version",
]
[[package]]
name = "windows"
version = "0.54.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"
dependencies = [
"windows-core 0.54.0",
"windows-targets 0.52.6",
]
[[package]]
name = "windows"
version = "0.61.3"
@@ -10490,6 +10788,16 @@ dependencies = [
"windows-core 0.62.2",
]
[[package]]
name = "windows-core"
version = "0.54.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"
dependencies = [
"windows-result 0.1.2",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-core"
version = "0.61.2"
@@ -10603,6 +10911,15 @@ dependencies = [
"windows-strings 0.4.2",
]
[[package]]
name = "windows-result"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -10985,7 +11302,7 @@ dependencies = [
"js-sys",
"libc",
"memmap2",
"ndk",
"ndk 0.9.0",
"objc2 0.5.2",
"objc2-app-kit 0.2.2",
"objc2-foundation 0.2.2",
@@ -11296,7 +11613,7 @@ dependencies = [
"javascriptcore-rs",
"jni 0.21.1",
"libc",
"ndk",
"ndk 0.9.0",
"objc2 0.6.4",
"objc2-app-kit 0.3.2",
"objc2-core-foundation",
+1
View File
@@ -31,6 +31,7 @@ open = "5"
reqwest = { workspace = true, features = ["multipart"] }
resvg = "0.45"
rfd = "0.16"
rodio = { version = "0.20", default-features = false, features = ["symphonia-all"] }
serde.workspace = true
serde_json.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
+22 -14
View File
@@ -7,8 +7,8 @@ The look follows the brutalist webapp (`artcraft-services/frontend/apps/artcraft
means near-black surfaces with white/15 hairlines and 3 px corners on controls. Type is Archivo for
display, Inter for body text and Geist Mono for labels. The brand blue marks the primary actions.
Create Image and Create Video are fully built. The other tools show a "coming soon" page that
links to the same tool in the web app.
Create Image, Create Video and Create Audio are fully built. The other tools show a "coming soon"
page that links to the same tool in the web app.
## Running
@@ -32,22 +32,24 @@ The workspace's default member is the Tauri app, so pass `-p artcraft_native`.
| `backend::wire` | Lenient response shapes; new server enum values can't break decoding |
| `backend::session` | Credentials: own file, Tauri cookie import, env config |
| `backend::media_cache` | Thumbnails and previews: downloaded and decoded off-thread, evicted when unused |
| `backend::audio` | One clip at a time: reference previews, feed tracks (pause, seek, volume), waveforms |
| `models` | Model catalog from OmniGen listings, plus names, families and limits |
| `prompt_box` | The shared prompt box: editor, reference deck, keyframes, pickers, model selector |
| `pages` | Create Image, Create Video, Home, Library, and the shared page shell |
| `pages` | Create Image, Video and Audio, Home, Library, and the shared page shell |
| `feed` | Generation feed: running, failed and finished jobs, library paging, grid and list |
| `overlays` | Lightbox, library picker, sign-in and settings dialogs |
| `shell` | Sidebar and top bar |
| `theme`, `ui` | Colour and type tokens, Lucide-style vector icons, widgets, toasts |
| `shell` | Custom title bar, sidebar flush to the window edge, content panel header |
| `theme`, `ui` | Tokens, Lucide-style icons, widgets, toasts, window chrome (drag, resize, caption buttons) |
How the backend works:
- **Generation.** It goes through OmniGen: `/v1/omni_gen/{models,cost,generate}/{image,video}`.
- **Generation.** It goes through OmniGen: `/v1/omni_gen/{models,cost,generate}/{image,video,audio}`.
Model capabilities come from the server, so new models and options appear without an app update.
- **Typed client functions.** Model lists, uploads, credits, login and deletes use
`artcraft_client`.
- **Lenient decoding.** Generate and cost requests are JSON maps, as in the Tauri app. Job polling
(`/v1/jobs/session`), the library list, batches and prompts are decoded with lenient structs.
(`/v1/jobs/session`), the audio model list, the library list, batches and prompts are decoded
with lenient structs.
- **Identity headers.** Every request sends the desktop identity headers: `Origin` and
`User-Agent`.
@@ -69,23 +71,29 @@ How the backend works:
| Live credit cost on the generate button | Done |
| Validation: starting frame, text-only banner, limits, uploads | Done |
| Clear all (confirms when references are attached) | Done |
| `@Character` mentions and the Characters modal (Seedance 2.0) | Not yet |
| `@Character` mentions and the Characters modal (create/edit/delete) | Done |
| Provider choice (Midjourney direct, fal) | Not yet; everything runs via ArtCraft |
| In-app video and audio playback | Opens in the system player instead |
| Animated video previews in the feed | Still frames only |
| In-app video playback | Done with FFmpeg on the PATH; otherwise opens in the system player |
| Audio reference playback (click to play/stop) | Done |
| Animated video previews in the feed (with a stills toggle) | Done |
| Audio: style, instrumental, keep lyrics, loop | Done |
| Audio: Beat & Key (BPM, musical key) and Tuning (sample rate, speed, volume, pitch) | Done |
| Audio references (play, 600 s cap) or one image, never both | Done |
Feed and lightbox:
- **Feed.** Pending cards show progress, time left and a batch banner. Failed cards show the reason
and can be dismissed. The grid is masonry, with a list view alongside. Hover actions are Recreate,
Make Video, Share and Download. Select mode does batch downloads. The last viewed item is marked.
- **Audio.** Audio cards play in place: a waveform (click to seek), the time and a volume flyout.
One track plays at a time and stops when you leave the page. Audio offers Share and Download.
- **Lightbox.** Prev/next with the arrow keys, prompt copy and details. Actions are Recreate, Make
Video, Download, Share and Delete.
Video, Download, Share and Delete. Audio plays in a panel with the full-size player.
## Diagnostics
Set `ARTCRAFT_SCREENSHOT=<file.png>` to save a screenshot once the window has settled.
`ARTCRAFT_PAGE` (`home`, `create-image`, `create-video` or `library`) picks the page, and
`ARTCRAFT_PAGE` (`home`, `create-image`, `create-video`, `create-audio` or `library`) picks the page, and
`ARTCRAFT_SCREENSHOT_EXIT=1` closes the app afterwards.
## Tests
@@ -95,8 +103,8 @@ cargo test -p artcraft_native
```
There are unit tests for the catalog, feed store, wire decoding and backend helpers. Headless UI
tests (`egui_kittest`) drive the prompt box: generate gating, Enter to generate, the pickers,
`@` mentions and clear-all.
tests (`egui_kittest`) drive the prompt box (generate gating, Enter to generate, the pickers, `@`
mentions, clear-all), the Characters dialog and the audio toolbar (toggles, Beat & Key, Tuning).
## Fonts and brand assets
+281 -140
View File
@@ -7,6 +7,7 @@ use std::time::Duration;
use egui::{Id, Ui};
use serde::{Deserialize, Serialize};
use crate::backend::audio::AudioPlayer;
use crate::backend::media_cache::MediaCache;
use crate::backend::wire::{Prompt, SessionUser};
use crate::backend::{Backend, Event, Modality, SHARE_URL_BASE};
@@ -14,24 +15,27 @@ use crate::feed::grid::{RatioCache, ViewMode};
use crate::feed::store::{FeedNotice, FeedStore};
use crate::feed::types::{FeedItem, MediaKind};
use crate::models::Catalog;
use crate::backend::wire::{Character, JobPhase};
use crate::overlays::account::{LoginAction, LoginDialog, SettingsAction, settings_dialog};
use crate::overlays::characters::{CharactersAction, CharactersModal};
use crate::overlays::library_picker::{LibraryPicker, PickerAction};
use crate::overlays::lightbox::{Lightbox, LightboxAction, fit_inside};
use crate::pages::common::{self, AppRequest, Env, FeedKey};
use crate::pages::create_audio::{AudioSettings, CreateAudioPage};
use crate::pages::create_image::{CreateImagePage, ImageSettings};
use crate::pages::create_video::{CreateVideoPage, VideoSettings};
use crate::pages::other;
use crate::prompt_box::refs::{RefKind, RefMedia};
use crate::shell::{self, Account, Page, ShellAction};
use crate::prompt_box::refs::{RefKind, RefMedia, References};
use crate::shell::{self, Account, FeedToggles, Page, ShellAction};
use crate::theme;
use crate::ui::toast::Toasts;
use crate::ui::widgets;
use crate::ui::{creator_icons, widgets, window};
/// Builds the page environment from the app's fields (disjoint borrows, so pages can be
/// borrowed mutably alongside it).
macro_rules! env {
($app:ident, $ctx:expr) => {
Env { ctx: $ctx.clone(), backend: &$app.backend, cache: &mut $app.cache, toasts: &mut $app.toasts, catalog: &$app.catalog, signed_in: $app.user.is_some(), enter_to_generate: $app.enter_to_generate, view_mode: $app.view_mode, ratios: &mut $app.ratios, prompts: &$app.prompts, requests: &mut $app.requests }
Env { ctx: $ctx.clone(), backend: &$app.backend, cache: &mut $app.cache, audio: &mut $app.player, toasts: &mut $app.toasts, catalog: &$app.catalog, signed_in: $app.user.is_some(), enter_to_generate: $app.enter_to_generate, view_mode: $app.view_mode, autoplay: $app.autoplay, ratios: &mut $app.ratios, prompts: &$app.prompts, requests: &mut $app.requests, characters: &$app.characters }
};
}
@@ -41,6 +45,8 @@ const CREDITS_INTERVAL: f64 = 60.0;
const MODEL_RETRY_INTERVAL: f64 = 15.0;
/// List view: prompt records fetched per frame at most.
const PROMPT_FETCHES_PER_FRAME: usize = 8;
/// Every create page's model listing.
const MODALITIES: [Modality; 3] = [Modality::Image, Modality::Video, Modality::Audio];
/// Preferences that survive restarts.
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -49,20 +55,23 @@ struct Saved {
page: Page,
enter_to_generate: bool,
view_mode: ViewMode,
autoplay: bool,
sidebar_open: bool,
image: ImageSettings,
video: VideoSettings,
audio: AudioSettings,
}
impl Default for Saved {
fn default() -> Self {
Self { page: Page::CreateImage, enter_to_generate: false, view_mode: ViewMode::Grid, sidebar_open: true, image: ImageSettings { count: 1, ..Default::default() }, video: VideoSettings::default() }
Self { page: Page::CreateImage, enter_to_generate: false, view_mode: ViewMode::Grid, autoplay: true, sidebar_open: true, image: ImageSettings { count: 1, ..Default::default() }, video: VideoSettings::default(), audio: AudioSettings::default() }
}
}
pub struct ArtcraftApp {
backend: Backend,
cache: MediaCache,
player: AudioPlayer,
toasts: Toasts,
catalog: Catalog,
user: Option<SessionUser>,
@@ -70,9 +79,11 @@ pub struct ArtcraftApp {
page: Page,
enter_to_generate: bool,
view_mode: ViewMode,
autoplay: bool,
sidebar_open: bool,
image: CreateImagePage,
video: CreateVideoPage,
audio: CreateAudioPage,
library: FeedStore,
ratios: RatioCache,
/// Prompt texts by prompt token (list view, lightbox).
@@ -86,9 +97,12 @@ pub struct ArtcraftApp {
lightbox: Option<Lightbox>,
picker: Option<LibraryPicker>,
login: Option<LoginDialog>,
characters_modal: Option<CharactersModal>,
characters: Vec<Character>,
/// Characters being made: (job token, name).
pending_characters: Vec<(String, String)>,
settings_open: bool,
preview: Option<String>,
logo: egui::TextureHandle,
requests: Vec<AppRequest>,
screenshot: Option<ScreenshotRequest>,
next_poll: f64,
@@ -106,10 +120,12 @@ impl ArtcraftApp {
}
let backend = Backend::new(cc.egui_ctx.clone());
let cache = MediaCache::new(backend.runtime(), backend.http());
backend.load_models(Modality::Image);
backend.load_models(Modality::Video);
let player = AudioPlayer::new(backend.runtime(), backend.http());
for modality in MODALITIES {
backend.load_models(modality);
}
backend.refresh_session();
Self { cache, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, settings_open: false, preview: None, logo: load_logo(&cc.egui_ctx), requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend }
Self { cache, player, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, autoplay: saved.autoplay, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), audio: CreateAudioPage::new(saved.audio), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, characters_modal: None, characters: Vec::new(), pending_characters: Vec::new(), settings_open: false, preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend }
}
// --- Events from the backend -------------------------------------------------------------
@@ -120,13 +136,16 @@ impl ArtcraftApp {
let changed = self.user.as_ref().map(|u| &u.username) != user.as_ref().map(|u| &u.username);
self.user = user;
if changed {
self.image.feed.reset();
self.video.feed.reset();
self.library.reset();
for feed in self.feeds_mut() {
feed.reset();
}
self.next_poll = 0.0;
}
if self.user.is_some() {
self.login = None;
if changed {
self.backend.load_characters();
}
} else {
self.credits = None;
if let Some(login) = &mut self.login {
@@ -150,6 +169,7 @@ impl ArtcraftApp {
Event::Models(modality, Ok(models)) => match modality {
Modality::Image => self.catalog.set_image(models),
Modality::Video => self.catalog.set_video(models),
Modality::Audio => self.catalog.set_audio(models),
},
Event::Models(_, Err(message)) => {
if self.models_retry_at.is_none() {
@@ -158,34 +178,65 @@ impl ArtcraftApp {
self.models_retry_at = Some(ctx.input(|i| i.time) + MODEL_RETRY_INTERVAL);
},
Event::Cost { key, credits } => {
self.image.cost.apply(&key, credits);
self.video.cost.apply(&key, credits);
for cost in [&mut self.image.cost, &mut self.video.cost, &mut self.audio.cost] {
cost.apply(&key, credits);
}
},
Event::Enqueued { modality, job_tokens, meta } => {
let mut env = env!(self, ctx);
match modality {
Modality::Image => self.image.on_enqueued(&mut env, &job_tokens, &meta),
Modality::Video => self.video.on_enqueued(&mut env, &job_tokens, &meta),
Modality::Audio => self.audio.on_enqueued(&mut env, &job_tokens, &meta),
}
self.next_poll = ctx.input(|i| i.time) + 2.0;
},
Event::EnqueueFailed { modality, message } => {
Event::EnqueueFailed { modality, message, status } => {
let mut env = env!(self, ctx);
match modality {
Modality::Image => self.image.on_enqueue_failed(&mut env, &message),
Modality::Video => self.video.on_enqueue_failed(&mut env, &message),
Modality::Audio => self.audio.on_enqueue_failed(&mut env, &message, status),
}
},
Event::Uploaded { ref_id, result } => {
if let Some(modal) = &mut self.characters_modal {
match modal.apply_upload(ref_id, result.clone()) {
Some(Err(err)) => {
self.toasts.error(format!("Failed to upload the reference image: {err}"));
return;
},
Some(Ok(())) => return,
None => {},
}
}
if self.image.refs.find_mut(ref_id).is_some() {
common::apply_upload(&mut self.toasts, &mut self.image.refs, ref_id, result);
} else if self.audio.refs.find_mut(ref_id).is_some() {
if common::apply_upload(&mut self.toasts, &mut self.audio.refs, ref_id, result) {
self.audio.check_durations(&mut self.toasts, ref_id);
}
} else if common::apply_upload(&mut self.toasts, &mut self.video.refs, ref_id, result) {
let mut env = env!(self, ctx);
self.video.check_durations(&mut env, ref_id);
}
},
Event::Jobs(jobs) => {
for (kind, notices) in [(MediaKind::Image, self.image.feed.apply_jobs(MediaKind::Image, &jobs, &self.catalog)), (MediaKind::Video, self.video.feed.apply_jobs(MediaKind::Video, &jobs, &self.catalog))] {
// Characters being made finish as jobs too.
let before = self.pending_characters.len();
let toasts = &mut self.toasts;
self.pending_characters.retain(|(job, name)| match jobs.iter().find(|j| &j.job_token == job).map(|j| j.status.phase()) {
Some(JobPhase::Succeeded) => false,
Some(JobPhase::Failed) => {
toasts.error(format!("Character \u{201c}{name}\u{201d} failed to create"));
false
},
_ => true,
});
if self.pending_characters.len() != before {
self.backend.load_characters();
}
for (kind, notices) in [(MediaKind::Image, self.image.feed.apply_jobs(MediaKind::Image, &jobs, &self.catalog)), (MediaKind::Video, self.video.feed.apply_jobs(MediaKind::Video, &jobs, &self.catalog)), (MediaKind::Audio, self.audio.feed.apply_jobs(MediaKind::Audio, &jobs, &self.catalog))] {
for notice in notices {
match notice {
FeedNotice::Completed => {
@@ -201,33 +252,30 @@ impl ArtcraftApp {
}
}
},
Event::LibraryPage { key, page, items, total_pages } => match key.as_str() {
"picker" => {
// The picker's pages come back under its own name.
Event::LibraryPage { key, page, items, total_pages } => match FeedKey::from_name(&key) {
Some(feed) => self.feed_mut(feed).apply_library_page(feed.kind(), page, &items, total_pages),
None => {
if let Some(p) = &mut self.picker {
p.apply_page(page, items, total_pages);
}
},
"image" => self.image.feed.apply_library_page(MediaKind::Image, page, &items, total_pages),
"video" => self.video.feed.apply_library_page(MediaKind::Video, page, &items, total_pages),
_ => self.library.apply_library_page(MediaKind::Image, page, &items, total_pages),
},
Event::LibraryFailed { key, message } => {
match key.as_str() {
"picker" => {
match FeedKey::from_name(&key) {
Some(feed) => self.feed_mut(feed).library_failed(),
None => {
if let Some(p) = &mut self.picker {
p.loading = false;
p.has_more = false;
}
},
"image" => self.image.feed.library_failed(),
"video" => self.video.feed.library_failed(),
_ => self.library.library_failed(),
}
self.toasts.error(format!("Couldn't load your library: {message}"));
},
Event::Batch { job_token, items } => match self.batch_kinds.remove(&job_token) {
Some(MediaKind::Video) => self.video.feed.apply_batch(MediaKind::Video, &items),
_ => self.image.feed.apply_batch(MediaKind::Image, &items),
Event::Batch { job_token, items } => {
let kind = self.batch_kinds.remove(&job_token).unwrap_or(MediaKind::Image);
self.feed_mut(FeedKey::of(kind)).apply_batch(kind, &items);
},
Event::Prompt(prompt) => {
if let Some(text) = &prompt.maybe_positive_prompt {
@@ -240,8 +288,15 @@ impl ArtcraftApp {
}
self.prompt_records.insert(prompt.token.clone(), prompt);
},
Event::Characters(list) => self.characters = list,
Event::CharacterCreating { job_token, name } => {
self.toasts.success(format!("Character \u{201c}{name}\u{201d} is being created"));
self.pending_characters.push((job_token, name));
self.next_poll = 0.0;
},
Event::CharacterChanged => self.backend.load_characters(),
Event::Deleted(token) => {
for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] {
for feed in self.feeds_mut() {
feed.remove_item(&token);
}
if self.lightbox.as_ref().is_some_and(|l| l.token == token) {
@@ -275,7 +330,7 @@ impl ArtcraftApp {
self.open_login();
return;
};
let refs = if page == MediaKind::Image { &self.image.refs } else { &self.video.refs };
let refs = self.refs(page);
let attached: HashSet<String> = refs.list(kind).iter().chain(refs.first_frame.iter()).chain(refs.last_frame.iter()).filter_map(|r| r.token.clone()).collect();
let mut picker = LibraryPicker::new(kind, slot, page, max, attached);
picker.loading = true;
@@ -288,8 +343,7 @@ impl ArtcraftApp {
};
let image = RefMedia::from_library(RefKind::Image, item.token.clone(), item.thumbnail.clone(), Some(item.full_url.clone()), 0.0);
self.video.set_start_image(image);
self.lightbox = None;
self.page = Page::CreateVideo;
self.navigate(Page::CreateVideo);
},
AppRequest::Recreate { kind, token } => {
let Some(prompt_token) = self.find_item(&token).and_then(|i| i.prompt_token.clone()) else {
@@ -315,17 +369,27 @@ impl ArtcraftApp {
},
AppRequest::Download(tokens) => self.download(&tokens),
AppRequest::Preview(url) => self.preview = Some(url),
AppRequest::ToggleAudio { ref_id, url } => self.player.toggle(ctx, &format!("ref:{ref_id}"), &url),
AppRequest::OpenCharacters => {
if self.user.is_none() {
self.open_login();
return;
}
self.backend.load_characters();
self.characters_modal = Some(CharactersModal::new());
},
AppRequest::PlayExternally(url) => {
if let Err(err) = open::that(&url) {
self.toasts.error(format!("Couldn't open the file: {err}"));
}
},
AppRequest::LoadMoreLibrary(key) => {
let Some(user) = &self.user else {
return;
};
let (feed, name) = match key {
FeedKey::Image => (&mut self.image.feed, "image"),
FeedKey::Video => (&mut self.video.feed, "video"),
FeedKey::Library => (&mut self.library, "library"),
};
if let Some(page) = feed.next_library_page() {
self.backend.load_library(name.to_owned(), user.username.clone(), key.media_classes(), page);
let username = user.username.clone();
if let Some(page) = self.feed_mut(key).next_library_page() {
self.backend.load_library(key.name().to_owned(), username, key.media_classes(), page);
}
},
}
@@ -335,14 +399,15 @@ impl ArtcraftApp {
match kind {
MediaKind::Image => {
self.image.apply_prompt(prompt);
self.page = Page::CreateImage;
self.navigate(Page::CreateImage);
},
MediaKind::Video => {
self.video.apply_prompt(prompt);
self.page = Page::CreateVideo;
self.navigate(Page::CreateVideo);
},
// Audio isn't recreated yet (no Recreate button offers it).
MediaKind::Audio => {},
}
self.lightbox = None;
}
fn download(&mut self, tokens: &[String]) {
@@ -360,7 +425,7 @@ impl ArtcraftApp {
for item in many {
self.backend.save_url(item.full_url.clone(), dir.join(file_name_for(item)));
}
for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] {
for feed in self.feeds_mut() {
feed.selecting = false;
feed.selected.clear();
}
@@ -370,7 +435,52 @@ impl ArtcraftApp {
}
fn find_item(&self, token: &str) -> Option<&FeedItem> {
self.image.feed.find(token).or_else(|| self.video.feed.find(token)).or_else(|| self.library.find(token))
self.feeds().into_iter().find_map(|f| f.find(token))
}
fn feeds(&self) -> [&FeedStore; 4] {
[&self.image.feed, &self.video.feed, &self.audio.feed, &self.library]
}
fn feeds_mut(&mut self) -> [&mut FeedStore; 4] {
[&mut self.image.feed, &mut self.video.feed, &mut self.audio.feed, &mut self.library]
}
fn feed(&self, key: FeedKey) -> &FeedStore {
match key {
FeedKey::Image => &self.image.feed,
FeedKey::Video => &self.video.feed,
FeedKey::Audio => &self.audio.feed,
FeedKey::Library => &self.library,
}
}
fn feed_mut(&mut self, key: FeedKey) -> &mut FeedStore {
match key {
FeedKey::Image => &mut self.image.feed,
FeedKey::Video => &mut self.video.feed,
FeedKey::Audio => &mut self.audio.feed,
FeedKey::Library => &mut self.library,
}
}
/// The references of the page that makes `kind`.
fn refs(&self, kind: MediaKind) -> &References {
match kind {
MediaKind::Image => &self.image.refs,
MediaKind::Video => &self.video.refs,
MediaKind::Audio => &self.audio.refs,
}
}
/// Opens `page`, closing the lightbox. Leaving a page stops its audio, like the webapp's
/// players unmounting.
fn navigate(&mut self, page: Page) {
if page != self.page {
self.player.stop();
}
self.page = page;
self.lightbox = None;
}
fn request_prompt(&mut self, prompt_token: Option<String>) {
@@ -387,12 +497,13 @@ impl ArtcraftApp {
}
}
/// The feed the top bar's toggles act on.
fn current_feed(&mut self) -> Option<&mut FeedStore> {
/// The feed the page shows (and the top bar's toggles act on).
fn page_feed(&self) -> Option<FeedKey> {
match self.page {
Page::CreateImage => Some(&mut self.image.feed),
Page::CreateVideo => Some(&mut self.video.feed),
Page::Library => Some(&mut self.library),
Page::CreateImage => Some(FeedKey::Image),
Page::CreateVideo => Some(FeedKey::Video),
Page::CreateAudio => Some(FeedKey::Audio),
Page::Library => Some(FeedKey::Library),
_ => None,
}
}
@@ -411,106 +522,116 @@ impl ArtcraftApp {
}
if self.models_retry_at.is_some_and(|t| now >= t) {
self.models_retry_at = None;
if self.catalog.image.is_empty() {
self.backend.load_models(Modality::Image);
}
if self.catalog.video.is_empty() {
self.backend.load_models(Modality::Video);
for (modality, loaded) in MODALITIES.into_iter().zip([&self.catalog.image, &self.catalog.video, &self.catalog.audio]) {
if loaded.is_empty() {
self.backend.load_models(modality);
}
}
}
ctx.request_repaint_after(Duration::from_secs(1));
}
/// The title bar and the sidebar (both on the window chrome).
fn shell(&mut self, ui: &mut Ui) {
let account = Account { username: self.user.as_ref().map(|u| u.username.as_str()), display_name: self.user.as_ref().map(|u| u.display_name.as_str()).filter(|n| !n.is_empty()), credits: self.credits };
let mark = creator_icons::texture(ui.ctx(), "artcraft");
let mut actions = Vec::new();
let chrome = egui::Frame::NONE.fill(theme::CHROME);
egui::Panel::top("titlebar").exact_size(shell::TITLEBAR_HEIGHT).resizable(false).frame(chrome).show(ui, |ui| {
actions.extend(shell::titlebar(ui, &account, &mark));
});
if self.sidebar_open {
egui::Panel::left("sidebar").exact_size(shell::sidebar_width()).resizable(false).frame(egui::Frame::NONE.fill(theme::BG)).show(ui, |ui| {
actions.extend(shell::sidebar(ui, self.page, &account, &self.logo));
egui::Panel::left("sidebar").exact_size(shell::SIDEBAR_WIDTH).resizable(false).frame(chrome).show(ui, |ui| {
actions.extend(shell::sidebar(ui, self.page, &account));
});
}
let toggles = match self.page {
Page::CreateImage => Some((self.view_mode, self.image.feed.selecting)),
Page::CreateVideo => Some((self.view_mode, self.video.feed.selecting)),
Page::Library => Some((self.view_mode, self.library.selecting)),
_ => None,
};
egui::Panel::top("topbar").exact_size(shell::topbar_height()).resizable(false).frame(egui::Frame::NONE).show(ui, |ui| {
actions.extend(shell::topbar(ui, self.page, &account, toggles, self.sidebar_open));
});
for action in actions {
match action {
ShellAction::Navigate(page) => {
self.page = page;
self.lightbox = None;
},
ShellAction::ToggleSidebar => self.sidebar_open = !self.sidebar_open,
ShellAction::SignIn => self.open_login(),
ShellAction::SignOut => {
self.backend.logout();
self.settings_open = false;
},
ShellAction::OpenSettings => self.settings_open = true,
ShellAction::SetViewMode(mode) => self.view_mode = mode,
ShellAction::ToggleSelect => {
if let Some(feed) = self.current_feed() {
feed.selecting = !feed.selecting;
feed.selected.clear();
}
},
}
self.apply_shell_action(action);
}
}
fn apply_shell_action(&mut self, action: ShellAction) {
match action {
ShellAction::Navigate(page) => self.navigate(page),
ShellAction::ToggleSidebar => self.sidebar_open = !self.sidebar_open,
ShellAction::SignIn => self.open_login(),
ShellAction::SignOut => {
self.backend.logout();
self.settings_open = false;
},
ShellAction::OpenSettings => self.settings_open = true,
ShellAction::SetViewMode(mode) => self.view_mode = mode,
ShellAction::SetAutoplay(on) => self.autoplay = on,
ShellAction::ToggleSelect => {
if let Some(key) = self.page_feed() {
let feed = self.feed_mut(key);
feed.selecting = !feed.selecting;
feed.selected.clear();
}
},
}
}
/// The content panel: inset from the window edges with rounded corners, its header on top
/// and the page below.
fn page_ui(&mut self, ui: &mut Ui) {
let ctx = ui.ctx().clone();
egui::CentralPanel::default().frame(egui::Frame::NONE.fill(theme::BG)).show(ui, |ui| {
let mut env = env!(self, ctx);
match self.page {
Page::Home => {
if let Some(page) = other::home(ui, env.signed_in, env.requests) {
self.page = page;
let gap = shell::PANEL_GAP;
let margin = egui::Margin { left: if self.sidebar_open { 0 } else { gap }, right: gap, top: 0, bottom: gap };
// Only feeds with videos offer the preview autoplay toggle.
let toggles = self.page_feed().map(|key| FeedToggles { mode: self.view_mode, selecting: self.feed(key).selecting, autoplay: matches!(key, FeedKey::Video | FeedKey::Library).then_some(self.autoplay) });
let mut header_action = None;
let mut open_page = None;
egui::CentralPanel::default().frame(egui::Frame::NONE.fill(theme::CHROME).inner_margin(margin)).show(ui, |ui| {
let panel = egui::Frame::new().fill(theme::BG).stroke(theme::hairline()).corner_radius(shell::PANEL_RADIUS);
panel.show(ui, |ui| {
ui.set_min_size(ui.available_size());
egui::Panel::top("page-header").exact_size(shell::HEADER_HEIGHT).resizable(false).frame(egui::Frame::NONE).show(ui, |ui| {
header_action = shell::page_header(ui, self.page, toggles);
});
egui::CentralPanel::default().frame(egui::Frame::NONE).show(ui, |ui| {
let mut env = env!(self, ctx);
match self.page {
Page::Home => open_page = other::home(ui, env.signed_in, env.requests),
Page::CreateImage => self.image.ui(ui, &mut env),
Page::CreateVideo => self.video.ui(ui, &mut env),
Page::CreateAudio => self.audio.ui(ui, &mut env),
Page::Library => other::library(ui, &mut env, &mut self.library),
page => other::coming_soon(ui, page),
}
},
Page::CreateImage => self.image.ui(ui, &mut env),
Page::CreateVideo => self.video.ui(ui, &mut env),
Page::Library => other::library(ui, &mut env, &mut self.library),
page => other::coming_soon(ui, page),
}
});
});
});
self.prefetch_list_prompts();
if let Some(action) = header_action {
self.apply_shell_action(action);
}
if let Some(page) = open_page {
self.navigate(page);
}
self.prefetch_prompts();
}
/// The list view shows prompt texts: fetch the records for the visible feed, a few at a time.
fn prefetch_list_prompts(&mut self) {
if self.view_mode != ViewMode::List {
/// The list view and audio cards show prompt texts: fetch the records for the visible feed, a
/// few at a time.
fn prefetch_prompts(&mut self) {
let Some(key) = self.page_feed() else {
return;
}
let feed = match self.page {
Page::CreateImage => &self.image.feed,
Page::CreateVideo => &self.video.feed,
Page::Library => &self.library,
_ => return,
};
let wanted: Vec<String> = feed.items.iter().filter_map(|i| i.prompt_token.clone()).filter(|t| !self.requested_prompts.contains(t)).take(PROMPT_FETCHES_PER_FRAME).collect();
let list = self.view_mode == ViewMode::List;
let wanted: Vec<String> = self.feed(key).items.iter().filter(|i| list || i.kind == MediaKind::Audio).filter_map(|i| i.prompt_token.clone()).filter(|t| !self.requested_prompts.contains(t)).take(PROMPT_FETCHES_PER_FRAME).collect();
for token in wanted {
self.request_prompt(Some(token));
}
}
fn overlays(&mut self, ctx: &egui::Context) {
if let Some(lb) = &mut self.lightbox {
let feed = match lb.kind {
_ if self.page == Page::Library => &self.library,
MediaKind::Image => &self.image.feed,
MediaKind::Video => &self.video.feed,
};
let order = feed.ordered_tokens();
let item = self.image.feed.find(&lb.token).or_else(|| self.video.feed.find(&lb.token)).or_else(|| self.library.find(&lb.token)).cloned();
match item {
if let Some((kind, token)) = self.lightbox.as_ref().map(|lb| (lb.kind, lb.token.clone())) {
let key = if self.page == Page::Library { FeedKey::Library } else { FeedKey::of(kind) };
let order = self.feed(key).ordered_tokens();
match self.find_item(&token).cloned() {
Some(item) => {
let prompt = item.prompt_token.as_ref().and_then(|t| self.prompt_records.get(t));
let actions = lb.show(ctx, &item, &order, prompt, &self.catalog, &mut self.cache);
let actions = self.lightbox.as_mut().map(|lb| lb.show(ctx, &item, &order, prompt, &self.catalog, &mut self.cache, &mut self.player)).unwrap_or_default();
for action in actions {
self.lightbox_action(ctx, action, &item);
}
@@ -532,8 +653,11 @@ impl ArtcraftApp {
},
PickerAction::Confirm(picks) => {
if let Some(p) = self.picker.take() {
let refs = if p.page == MediaKind::Image { &mut self.image.refs } else { &mut self.video.refs };
common::attach_from_library(refs, p.kind, p.slot, picks);
match p.page {
MediaKind::Image => common::attach_from_library(&mut self.image.refs, p.kind, p.slot, picks),
MediaKind::Video => common::attach_from_library(&mut self.video.refs, p.kind, p.slot, picks),
MediaKind::Audio => self.audio.attach_from_library(&mut self.toasts, &self.catalog, p.kind, picks),
}
}
},
}
@@ -561,6 +685,27 @@ impl ArtcraftApp {
None => {},
}
}
if let Some(modal) = &mut self.characters_modal {
let pending: Vec<String> = self.pending_characters.iter().map(|(_, n)| n.clone()).collect();
for action in modal.show(ctx, &self.characters, &pending, &mut self.cache) {
match action {
CharactersAction::Close => self.characters_modal = None,
CharactersAction::Select { token, name } => {
self.video.mention_character(&name, &token);
self.characters_modal = None;
},
CharactersAction::Upload { ref_id, path } => self.backend.upload_file(ref_id, RefKind::Image, path),
CharactersAction::Create { image_token, name, description } => self.backend.create_character(image_token, name, description),
CharactersAction::Edit { token, name, description } => self.backend.edit_character(token, name, description),
CharactersAction::Delete { token, name } => self.backend.delete_character(token, name),
CharactersAction::Preview(url) => self.preview = Some(url),
CharactersAction::Toast(message) => self.toasts.error(message),
}
if self.characters_modal.is_none() {
break;
}
}
}
if self.settings_open {
let username = self.user.as_ref().map(|u| u.username.clone());
match settings_dialog(ctx, &mut self.enter_to_generate, username.as_deref(), self.credits, self.backend.data_root()) {
@@ -602,7 +747,7 @@ impl ArtcraftApp {
if let Some(next) = self.find_item(&token).cloned() {
self.request_prompt(next.prompt_token);
}
for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] {
for feed in self.feeds_mut() {
if feed.find(&token).is_some() {
feed.last_viewed = Some(token.clone());
}
@@ -640,6 +785,11 @@ impl eframe::App for ArtcraftApp {
fn ui(&mut self, ui: &mut Ui, _frame: &mut eframe::Frame) {
let ctx = ui.ctx().clone();
self.cache.poll(&ctx);
if let Some(err) = self.player.poll() {
self.toasts.error(err);
}
let playing = [&self.video.refs, &self.image.refs].iter().flat_map(|r| r.audios.iter()).map(|r| r.id).find(|id| self.player.is_playing(&format!("ref:{id}")));
crate::prompt_box::deck::set_playing_audio(&ctx, playing);
self.shell(ui);
self.page_ui(ui);
for request in std::mem::take(&mut self.requests) {
@@ -647,18 +797,19 @@ impl eframe::App for ArtcraftApp {
}
self.overlays(&ctx);
self.toasts.show(&ctx);
window::window_edges(&ctx);
if let Some(shot) = &mut self.screenshot {
shot.tick(&ctx);
}
}
fn save(&mut self, storage: &mut dyn eframe::Storage) {
let saved = Saved { page: self.page, enter_to_generate: self.enter_to_generate, view_mode: self.view_mode, sidebar_open: self.sidebar_open, image: self.image.settings.clone(), video: self.video.settings.clone() };
let saved = Saved { page: self.page, enter_to_generate: self.enter_to_generate, view_mode: self.view_mode, autoplay: self.autoplay, sidebar_open: self.sidebar_open, image: self.image.settings.clone(), video: self.video.settings.clone(), audio: self.audio.settings.clone() };
eframe::set_value(storage, STORAGE_KEY, &saved);
}
fn clear_color(&self, _visuals: &egui::Visuals) -> [f32; 4] {
theme::BG.to_normalized_gamma_f32()
theme::CHROME.to_normalized_gamma_f32()
}
}
@@ -709,21 +860,11 @@ impl ScreenshotRequest {
/// `artcraft_<token>.<ext>`, the extension taken from the URL.
fn file_name_for(item: &FeedItem) -> String {
let ext = item.full_url.rsplit('.').next().filter(|e| e.len() <= 4 && e.chars().all(|c| c.is_ascii_alphanumeric())).unwrap_or(if item.kind == MediaKind::Video { "mp4" } else { "png" });
let fallback = match item.kind {
MediaKind::Image => "png",
MediaKind::Video => "mp4",
MediaKind::Audio => "mp3",
};
let ext = item.full_url.rsplit('.').next().filter(|e| e.len() <= 4 && e.chars().all(|c| c.is_ascii_alphanumeric())).unwrap_or(fallback);
format!("artcraft_{}.{ext}", item.token)
}
/// The ArtCraft wordmark (the Tauri app's white-on-dark SVG), rasterized for the sidebar.
fn load_logo(ctx: &egui::Context) -> egui::TextureHandle {
const LOGO: &[u8] = include_bytes!("../../frontend/apps/artcraft/app/public/resources/logo/artcraft-logo-color-white.svg");
/// Twice the 20 pt the sidebar draws it at, for high-DPI screens.
const HEIGHT_PX: f32 = 48.0;
let image = resvg::usvg::Tree::from_data(LOGO, &resvg::usvg::Options::default()).ok().and_then(|tree| {
let scale = HEIGHT_PX / tree.size().height();
let (w, h) = ((tree.size().width() * scale).ceil() as u32, HEIGHT_PX as u32);
let mut pixmap = resvg::tiny_skia::Pixmap::new(w, h)?;
resvg::render(&tree, resvg::tiny_skia::Transform::from_scale(scale, scale), &mut pixmap.as_mut());
Some(egui::ColorImage::from_rgba_premultiplied([w as usize, h as usize], pixmap.data()))
});
ctx.load_texture("artcraft-logo", image.unwrap_or_else(|| egui::ColorImage::filled([1, 1], egui::Color32::TRANSPARENT)), egui::TextureOptions::LINEAR)
}
+343
View File
@@ -0,0 +1,343 @@
//! Audio, one clip at a time (`audio-playback-controller`): reference cards play and stop on
//! click (the webapp's `DeckCard`), feed tracks pause, resume and seek (`WaveformAudioPlayer`).
//! Tracks also get waveforms, decoded off the UI thread. The output device opens on first use;
//! files download in the background.
use std::collections::HashMap;
use std::io::Cursor;
use std::sync::Arc;
use std::sync::mpsc::{Receiver, Sender, channel};
use std::time::Duration;
use log::warn;
use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink, Source};
use tokio::sync::Semaphore;
/// The webapp's reference preview volume.
const PREVIEW_VOLUME: f32 = 0.2;
/// Peaks kept per waveform (resampled to however many bars are drawn).
const PEAK_COUNT: usize = 256;
/// Frames folded into one peak while decoding.
const WINDOW_FRAMES: usize = 1024;
/// Waveforms decoding at once (each downloads its whole file).
const MAX_CONCURRENT_WAVEFORMS: usize = 2;
pub struct AudioPlayer {
rt: tokio::runtime::Handle,
http: reqwest::Client,
/// Opened on first play (and kept: closing it would stop the sound).
output: Option<(OutputStream, OutputStreamHandle)>,
sink: Option<Sink>,
/// What's playing (or loading).
current: Option<Clip>,
tx: Sender<Loaded>,
rx: Receiver<Loaded>,
waveforms: HashMap<String, WaveformLookup>,
limiter: Arc<Semaphore>,
/// The tracks' volume slider, 0–1.
track_volume: f32,
}
/// A track's shape: peaks (0–1, loudest = 1) and its length in seconds.
#[derive(Debug, PartialEq)]
pub struct Waveform {
pub peaks: Vec<f32>,
pub duration: f32,
}
/// What [`AudioPlayer::waveform`] knows about a track.
#[derive(Clone)]
pub enum WaveformLookup {
Loading,
Ready(Arc<Waveform>),
Failed,
}
/// Where the current track is.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Playback {
/// Still downloading.
pub loading: bool,
pub paused: bool,
/// Seconds in.
pub position: f32,
}
/// The clip that's playing, by the caller's key (a track's key is its URL).
struct Clip {
key: String,
track: bool,
/// Where to start once it has downloaded (a seek before it loaded).
start_at: f32,
}
enum Loaded {
Clip { key: String, bytes: Result<Arc<[u8]>, String> },
Waveform { url: String, waveform: Option<Waveform> },
}
impl AudioPlayer {
pub fn new(rt: tokio::runtime::Handle, http: reqwest::Client) -> Self {
let (tx, rx) = channel();
Self { rt, http, output: None, sink: None, current: None, tx, rx, waveforms: HashMap::new(), limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_WAVEFORMS)), track_volume: 1.0 }
}
/// Starts a reference clip under `key`, or stops it if it's what's playing.
pub fn toggle(&mut self, ctx: &egui::Context, key: &str, url: &str) {
if self.is_playing(key) {
self.stop();
} else {
self.start(ctx, key, url, false, 0.0);
}
}
/// Plays or pauses a feed track.
pub fn toggle_track(&mut self, ctx: &egui::Context, url: &str) {
match (&self.sink, self.is_playing(url)) {
(Some(sink), true) if sink.is_paused() => sink.play(),
(Some(sink), true) => sink.pause(),
// Still downloading: a second click cancels.
(None, true) => self.stop(),
_ => self.start(ctx, url, url, true, 0.0),
}
}
/// Jumps a feed track to `secs` (starting it there if it isn't the current one).
pub fn seek_track(&mut self, ctx: &egui::Context, url: &str, secs: f32) {
if !self.is_playing(url) {
self.start(ctx, url, url, true, secs);
return;
}
match &self.sink {
Some(sink) => {
if let Err(err) = sink.try_seek(Duration::from_secs_f32(secs.max(0.0))) {
warn!("Audio seek failed: {err}");
}
sink.play();
},
None => {
if let Some(clip) = &mut self.current {
clip.start_at = secs;
}
},
}
}
pub fn stop(&mut self) {
if let Some(sink) = self.sink.take() {
sink.stop();
}
self.current = None;
}
/// Whether `key` is playing, paused or about to play.
pub fn is_playing(&self, key: &str) -> bool {
self.current.as_ref().is_some_and(|c| c.key == key)
}
/// Where the track at `url` is, when it's the current one.
pub fn track(&self, url: &str) -> Option<Playback> {
let clip = self.current.as_ref().filter(|c| c.key == url)?;
Some(match &self.sink {
Some(sink) => Playback { loading: false, paused: sink.is_paused(), position: sink.get_pos().as_secs_f32() },
None => Playback { loading: true, paused: false, position: clip.start_at },
})
}
pub fn track_volume(&self) -> f32 {
self.track_volume
}
pub fn set_track_volume(&mut self, volume: f32) {
self.track_volume = volume.clamp(0.0, 1.0);
if let (Some(sink), Some(clip)) = (&self.sink, &self.current) {
if clip.track {
sink.set_volume(self.track_volume);
}
}
}
/// The waveform of the track at `url`, downloading and decoding it the first time it's asked
/// for.
pub fn waveform(&mut self, ctx: &egui::Context, url: &str) -> WaveformLookup {
if let Some(lookup) = self.waveforms.get(url) {
return lookup.clone();
}
self.waveforms.insert(url.to_owned(), WaveformLookup::Loading);
let (http, limiter, tx, ctx, url) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone(), url.to_owned());
self.rt.spawn(async move {
let waveform = match limiter.acquire_owned().await {
Ok(_permit) => match download(&http, &url).await {
Ok(bytes) => tokio::task::spawn_blocking(move || decode_waveform(bytes)).await.ok().flatten(),
Err(err) => {
warn!("Waveform download failed for {url}: {err}");
None
},
},
Err(_) => None,
};
let _ = tx.send(Loaded::Waveform { url, waveform });
ctx.request_repaint();
});
WaveformLookup::Loading
}
/// Starts downloads that finished and notices clips that ended. Call once per frame. Returns an
/// error message when a clip couldn't play.
pub fn poll(&mut self) -> Option<String> {
let mut error = None;
while let Ok(loaded) = self.rx.try_recv() {
match loaded {
Loaded::Waveform { url, waveform } => {
self.waveforms.insert(url, waveform.map_or(WaveformLookup::Failed, |w| WaveformLookup::Ready(Arc::new(w))));
},
Loaded::Clip { key, bytes } => {
// Superseded by another clip (or stopped) while downloading.
let Some(clip) = self.current.as_ref().filter(|c| c.key == key) else {
continue;
};
let (volume, start_at) = (if clip.track { self.track_volume } else { PREVIEW_VOLUME }, clip.start_at);
if let Err(err) = bytes.and_then(|b| self.play_bytes(b, volume, start_at)) {
warn!("Audio playback failed: {err}");
self.current = None;
error = Some(format!("Couldn't play the audio: {err}"));
}
},
}
}
if self.sink.as_ref().is_some_and(Sink::empty) {
self.sink = None;
self.current = None;
}
error
}
fn start(&mut self, ctx: &egui::Context, key: &str, url: &str, track: bool, start_at: f32) {
self.stop();
self.current = Some(Clip { key: key.to_owned(), track, start_at });
let (http, tx, ctx, key, url) = (self.http.clone(), self.tx.clone(), ctx.clone(), key.to_owned(), url.to_owned());
self.rt.spawn(async move {
let bytes = download(&http, &url).await;
let _ = tx.send(Loaded::Clip { key, bytes });
ctx.request_repaint();
});
}
fn play_bytes(&mut self, bytes: Arc<[u8]>, volume: f32, start_at: f32) -> Result<(), String> {
if self.output.is_none() {
self.output = Some(OutputStream::try_default().map_err(|e| e.to_string())?);
}
let Some((_, handle)) = &self.output else {
return Err("no audio output".to_owned());
};
let sink = Sink::try_new(handle).map_err(|e| e.to_string())?;
let source = Decoder::new(Cursor::new(bytes)).map_err(|e| e.to_string())?;
sink.set_volume(volume);
sink.append(source);
if start_at > 0.0 {
if let Err(err) = sink.try_seek(Duration::from_secs_f32(start_at)) {
warn!("Audio seek failed: {err}");
}
}
self.sink = Some(sink);
Ok(())
}
}
async fn download(http: &reqwest::Client, url: &str) -> Result<Arc<[u8]>, String> {
let response = http.get(url).send().await.and_then(|r| r.error_for_status()).map_err(|e| e.to_string())?;
response.bytes().await.map(|b| Arc::from(b.as_ref())).map_err(|e| e.to_string())
}
/// Decodes a whole file into its waveform: the loudest sample of each window, folded down to
/// [`PEAK_COUNT`] peaks and scaled so the loudest is 1.
fn decode_waveform(bytes: Arc<[u8]>) -> Option<Waveform> {
let decoder = match Decoder::new(Cursor::new(bytes)) {
Ok(d) => d,
Err(err) => {
warn!("Waveform decode failed: {err}");
return None;
},
};
let (channels, rate) = (usize::from(decoder.channels().max(1)), decoder.sample_rate().max(1));
let window = WINDOW_FRAMES * channels;
let (mut windows, mut loudest, mut count) = (Vec::new(), 0_u16, 0_usize);
for sample in decoder {
loudest = loudest.max(sample.unsigned_abs());
count += 1;
if count % window == 0 {
windows.push(f32::from(loudest));
loudest = 0;
}
}
if count % window != 0 {
windows.push(f32::from(loudest));
}
if count == 0 {
return None;
}
Some(Waveform { peaks: normalized(&resample(&windows, PEAK_COUNT)), duration: (count / channels) as f32 / rate as f32 })
}
/// `values` folded (by maximum) or stretched to exactly `count` entries.
pub fn resample(values: &[f32], count: usize) -> Vec<f32> {
if values.is_empty() || count == 0 {
return vec![0.0; count];
}
(0..count)
.map(|i| {
let start = i * values.len() / count;
let end = ((i + 1) * values.len() / count).max(start + 1).min(values.len());
values[start..end].iter().copied().fold(0.0, f32::max)
})
.collect()
}
fn normalized(values: &[f32]) -> Vec<f32> {
let max = values.iter().copied().fold(0.0, f32::max);
if max <= 0.0 {
return values.to_vec();
}
values.iter().map(|v| v / max).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resampling_keeps_the_loudest_of_each_span() {
assert_eq!(resample(&[0.1, 0.9, 0.2, 0.4], 2), [0.9, 0.4]);
assert_eq!(resample(&[0.5, 1.0], 4), [0.5, 0.5, 1.0, 1.0], "short inputs stretch");
assert_eq!(resample(&[], 3), [0.0, 0.0, 0.0]);
assert_eq!(normalized(&[0.0, 2.0, 1.0]), [0.0, 1.0, 0.5]);
}
#[test]
fn decodes_a_wav_into_peaks_and_duration() {
// One second of 8 kHz mono: silence, then a loud half.
let rate = 8000_u32;
let samples: Vec<i16> = (0..rate).map(|i| if i < rate / 2 { 0 } else { 16000 }).collect();
let mut wav = Vec::new();
wav.extend_from_slice(b"RIFF");
wav.extend_from_slice(&(36 + samples.len() as u32 * 2).to_le_bytes());
wav.extend_from_slice(b"WAVEfmt ");
wav.extend_from_slice(&16_u32.to_le_bytes());
wav.extend_from_slice(&1_u16.to_le_bytes());
wav.extend_from_slice(&1_u16.to_le_bytes());
wav.extend_from_slice(&rate.to_le_bytes());
wav.extend_from_slice(&(rate * 2).to_le_bytes());
wav.extend_from_slice(&2_u16.to_le_bytes());
wav.extend_from_slice(&16_u16.to_le_bytes());
wav.extend_from_slice(b"data");
wav.extend_from_slice(&(samples.len() as u32 * 2).to_le_bytes());
for s in &samples {
wav.extend_from_slice(&s.to_le_bytes());
}
let waveform = decode_waveform(Arc::from(wav)).expect("decodes");
assert!((waveform.duration - 1.0).abs() < 0.01, "duration {}", waveform.duration);
assert_eq!(waveform.peaks.len(), PEAK_COUNT);
assert_eq!(waveform.peaks[0], 0.0, "starts silent");
assert_eq!(*waveform.peaks.last().unwrap(), 1.0, "ends loud");
}
}
+132 -41
View File
@@ -1,14 +1,18 @@
//! Images shown in the UI: CDN thumbnails and local previews, as GPU textures.
//! Images shown in the UI: CDN thumbnails, animated video previews and local previews, as GPU
//! textures.
//!
//! Downloads run on the tokio runtime (a few at a time) and decode off the UI thread; the UI
//! uploads finished images in [`MediaCache::poll`]. Textures nobody has drawn for a while are
//! dropped once the cache grows past its budget, so long feeds don't hold every image forever.
use std::collections::HashMap;
use std::io::Cursor;
use std::sync::Arc;
use std::sync::mpsc::{Receiver, Sender, channel};
use std::time::Duration;
use egui::{ColorImage, TextureHandle, TextureOptions};
use image::AnimationDecoder;
use log::warn;
use tokio::runtime::Handle;
use tokio::sync::Semaphore;
@@ -17,10 +21,14 @@ use tokio::sync::Semaphore;
const MAX_CONCURRENT_DOWNLOADS: usize = 6;
/// Longest side of a decoded image; larger ones are downscaled before upload.
const MAX_DECODED_SIDE: u32 = 2048;
/// Animated previews: longest side and frame budget (each frame is its own texture).
const ANIMATED_SIDE: u32 = 320;
const ANIMATED_MAX_FRAMES: usize = 48;
/// Textures kept even when unused (roughly two screens of thumbnails).
const SOFT_CAPACITY: usize = 160;
/// Frames an image may go undrawn before it can be evicted.
/// Frames an image may go undrawn before it can be evicted (animations go sooner: they're heavy).
const EVICT_AFTER_FRAMES: u64 = 600;
const EVICT_ANIMATED_AFTER_FRAMES: u64 = 120;
pub struct MediaCache {
rt: Handle,
@@ -40,14 +48,30 @@ pub enum Lookup {
}
enum Entry {
Loading { last_used: u64 },
Ready { texture: TextureHandle, last_used: u64 },
Loading {
last_used: u64,
},
Ready {
texture: TextureHandle,
last_used: u64,
},
/// Frames with how long each shows (seconds), and their total.
Animated {
frames: Vec<(TextureHandle, f32)>,
total: f32,
last_used: u64,
},
Failed,
}
struct Loaded {
key: String,
image: Option<ColorImage>,
decoded: Option<Decoded>,
}
enum Decoded {
Still(ColorImage),
Frames(Vec<(ColorImage, f32)>),
}
impl MediaCache {
@@ -58,23 +82,13 @@ impl MediaCache {
/// The texture for `url`, starting its download the first time it's asked for.
pub fn get(&mut self, ctx: &egui::Context, url: &str) -> Lookup {
let frame = self.frame;
match self.entries.get_mut(url) {
Some(Entry::Ready { texture, last_used }) => {
*last_used = frame;
Lookup::Ready(texture.clone())
},
Some(Entry::Loading { last_used }) => {
*last_used = frame;
Lookup::Loading
},
Some(Entry::Failed) => Lookup::Failed,
None => {
self.entries.insert(url.to_owned(), Entry::Loading { last_used: frame });
self.spawn_download(ctx, url.to_owned());
Lookup::Loading
},
}
self.lookup(ctx, url, url, false)
}
/// The current frame of an animated image (a video's animated preview); still images come
/// back as they are.
pub fn get_animated(&mut self, ctx: &egui::Context, url: &str) -> Lookup {
self.lookup(ctx, &format!("anim:{url}"), url, true)
}
/// Decodes `bytes` off the UI thread and registers the result under `key`.
@@ -85,8 +99,8 @@ impl MediaCache {
self.entries.insert(key.to_owned(), Entry::Loading { last_used: self.frame });
let (tx, ctx, key) = (self.tx.clone(), ctx.clone(), key.to_owned());
self.rt.spawn_blocking(move || {
let image = decode(&bytes);
let _ = tx.send(Loaded { key, image });
let decoded = decode(&bytes).map(Decoded::Still);
let _ = tx.send(Loaded { key, decoded });
ctx.request_repaint();
});
}
@@ -94,39 +108,78 @@ impl MediaCache {
/// Uploads finished downloads and evicts stale textures. Call once per frame.
pub fn poll(&mut self, ctx: &egui::Context) {
self.frame += 1;
while let Ok(Loaded { key, image }) = self.rx.try_recv() {
while let Ok(Loaded { key, decoded }) = self.rx.try_recv() {
// Dropped while loading (evicted): don't resurrect it.
let Some(entry) = self.entries.get_mut(&key) else {
continue;
};
*entry = match image {
Some(image) => Entry::Ready { texture: ctx.load_texture(&key, image, TextureOptions::LINEAR), last_used: self.frame },
*entry = match decoded {
Some(Decoded::Still(image)) => Entry::Ready { texture: ctx.load_texture(&key, image, TextureOptions::LINEAR), last_used: self.frame },
Some(Decoded::Frames(frames)) => {
let total = frames.iter().map(|(_, d)| d).sum();
let frames = frames.into_iter().enumerate().map(|(i, (img, d))| (ctx.load_texture(format!("{key}#{i}"), img, TextureOptions::LINEAR), d)).collect();
Entry::Animated { frames, total, last_used: self.frame }
},
None => Entry::Failed,
};
}
if self.entries.len() > SOFT_CAPACITY {
let frame = self.frame;
self.entries.retain(|_, e| match e {
Entry::Ready { last_used, .. } | Entry::Loading { last_used } => frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES,
Entry::Failed => true,
});
let frame = self.frame;
let crowded = self.entries.len() > SOFT_CAPACITY;
self.entries.retain(|_, e| match e {
Entry::Animated { last_used, .. } => frame.saturating_sub(*last_used) < EVICT_ANIMATED_AFTER_FRAMES,
Entry::Ready { last_used, .. } | Entry::Loading { last_used } => !crowded || frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES,
Entry::Failed => true,
});
}
fn lookup(&mut self, ctx: &egui::Context, key: &str, url: &str, animated: bool) -> Lookup {
let frame = self.frame;
match self.entries.get_mut(key) {
Some(Entry::Ready { texture, last_used }) => {
*last_used = frame;
Lookup::Ready(texture.clone())
},
Some(Entry::Animated { frames, total, last_used }) => {
*last_used = frame;
let t = (ctx.input(|i| i.time) as f32).rem_euclid(total.max(0.01));
let (mut acc, mut index) = (0.0, frames.len() - 1);
for (i, (_, d)) in frames.iter().enumerate() {
acc += d;
if t < acc {
index = i;
break;
}
}
ctx.request_repaint_after(Duration::from_secs_f32((acc - t).max(0.01)));
Lookup::Ready(frames[index].0.clone())
},
Some(Entry::Loading { last_used }) => {
*last_used = frame;
Lookup::Loading
},
Some(Entry::Failed) => Lookup::Failed,
None => {
self.entries.insert(key.to_owned(), Entry::Loading { last_used: frame });
self.spawn_download(ctx, key.to_owned(), url.to_owned(), animated);
Lookup::Loading
},
}
}
fn spawn_download(&self, ctx: &egui::Context, url: String) {
fn spawn_download(&self, ctx: &egui::Context, key: String, url: String, animated: bool) {
let (http, limiter, tx, ctx) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone());
self.rt.spawn(async move {
let image = match limiter.acquire_owned().await {
Ok(_permit) => download(&http, &url).await,
let decoded = match limiter.acquire_owned().await {
Ok(_permit) => download(&http, &url, animated).await,
Err(_) => None,
};
let _ = tx.send(Loaded { key: url, image });
let _ = tx.send(Loaded { key, decoded });
ctx.request_repaint();
});
}
}
async fn download(http: &reqwest::Client, url: &str) -> Option<ColorImage> {
async fn download(http: &reqwest::Client, url: &str, animated: bool) -> Option<Decoded> {
let response = match http.get(url).send().await.and_then(|r| r.error_for_status()) {
Ok(response) => response,
Err(err) => {
@@ -135,7 +188,7 @@ async fn download(http: &reqwest::Client, url: &str) -> Option<ColorImage> {
},
};
let bytes = response.bytes().await.ok()?;
tokio::task::spawn_blocking(move || decode(&bytes)).await.ok().flatten()
tokio::task::spawn_blocking(move || if animated { decode_frames(&bytes).map(Decoded::Frames).or_else(|| decode(&bytes).map(Decoded::Still)) } else { decode(&bytes).map(Decoded::Still) }).await.ok().flatten()
}
fn decode(bytes: &[u8]) -> Option<ColorImage> {
@@ -147,7 +200,45 @@ fn decode(bytes: &[u8]) -> Option<ColorImage> {
},
};
let image = if image.width().max(image.height()) > MAX_DECODED_SIDE { image.thumbnail(MAX_DECODED_SIDE, MAX_DECODED_SIDE) } else { image };
let rgba = image.to_rgba8();
Some(to_color_image(image.to_rgba8()))
}
/// The frames of an animated GIF or WebP (subsampled to the budget), or `None` when it isn't
/// animated.
fn decode_frames(bytes: &[u8]) -> Option<Vec<(ColorImage, f32)>> {
let frames = match image::guess_format(bytes).ok()? {
image::ImageFormat::Gif => image::codecs::gif::GifDecoder::new(Cursor::new(bytes)).ok()?.into_frames().collect_frames().ok()?,
image::ImageFormat::WebP => {
let decoder = image::codecs::webp::WebPDecoder::new(Cursor::new(bytes)).ok()?;
if !decoder.has_animation() {
return None;
}
decoder.into_frames().collect_frames().ok()?
},
_ => return None,
};
if frames.len() < 2 {
return None;
}
// Keep every n-th frame, folding the skipped frames' time into the kept one.
let step = frames.len().div_ceil(ANIMATED_MAX_FRAMES);
let mut out = Vec::with_capacity(frames.len() / step + 1);
for chunk in frames.chunks(step) {
let secs: f32 = chunk
.iter()
.map(|f| {
let (n, d) = f.delay().numer_denom_ms();
n as f32 / d.max(1) as f32 / 1000.0
})
.sum();
let image = image::DynamicImage::ImageRgba8(chunk[0].buffer().clone()).thumbnail(ANIMATED_SIDE, ANIMATED_SIDE);
// Some encoders write 0 ms delays; browsers treat those as ~100 ms.
out.push((to_color_image(image.to_rgba8()), if secs < 0.02 { 0.1 * chunk.len() as f32 } else { secs }));
}
Some(out)
}
fn to_color_image(rgba: image::RgbaImage) -> ColorImage {
let size = [rgba.width() as usize, rgba.height() as usize];
Some(ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()))
ColorImage::from_rgba_unmultiplied(size, rgba.as_raw())
}
+141 -10
View File
@@ -1,8 +1,10 @@
//! Everything that talks to the ArtCraft API. Calls run on a tokio runtime and report back as
//! [`Event`]s, which the UI drains once per frame; nothing here blocks the UI thread.
pub mod audio;
pub mod media_cache;
pub mod session;
pub mod video;
pub mod wire;
use std::future::Future;
@@ -36,7 +38,7 @@ use tokens::tokens::media_files::MediaFileToken;
use crate::feed::types::MediaKind;
use crate::models::{self, ModelInfo};
use session::Credentials;
use wire::{BatchMedia, MediaFile, PromptResponse, SessionInfo, SessionJobs, SessionUser, UploadResponse, UserMediaList};
use wire::{AudioModels, BatchMedia, CharactersPage, MediaFile, PromptResponse, SessionInfo, SessionJobs, SessionUser, UploadResponse, UserMediaList};
/// Items per library page (`useGalleryData`'s `PAGE_SIZE`).
pub const LIBRARY_PAGE_SIZE: u32 = 40;
@@ -45,6 +47,8 @@ const MODEL_LIST_ATTEMPTS: u32 = 3;
const MODEL_LIST_BACKOFF: Duration = Duration::from_millis(250);
/// The share link base (`SHARE_URL_BASE`).
pub const SHARE_URL_BASE: &str = "https://getartcraft.com/media/";
/// Characters are made for this model (the Tauri app's `CharactersModal`).
const CHARACTER_MODEL: &str = "seedance_2p0";
/// Sent on every request (`artcraft_client_identity::ARTCRAFT_DESKTOP_USER_AGENT`).
const USER_AGENT: &str = "storyteller-client/1.0";
@@ -53,6 +57,7 @@ const USER_AGENT: &str = "storyteller-client/1.0";
pub enum Modality {
Image,
Video,
Audio,
}
impl Modality {
@@ -60,6 +65,7 @@ impl Modality {
match self {
Modality::Image => "image",
Modality::Video => "video",
Modality::Audio => "audio",
}
}
}
@@ -69,6 +75,7 @@ impl From<MediaKind> for Modality {
match kind {
MediaKind::Image => Modality::Image,
MediaKind::Video => Modality::Video,
MediaKind::Audio => Modality::Audio,
}
}
}
@@ -98,6 +105,8 @@ pub enum Event {
EnqueueFailed {
modality: Modality,
message: String,
/// The HTTP status, when the server answered.
status: Option<u16>,
},
Uploaded {
ref_id: u64,
@@ -119,6 +128,14 @@ pub enum Event {
items: Vec<MediaFile>,
},
Prompt(wire::Prompt),
/// The user's characters, newest first.
Characters(Vec<wire::Character>),
/// A character is being made by job `job_token`.
CharacterCreating {
job_token: String,
name: String,
},
CharacterChanged,
Deleted(String),
Saved(PathBuf),
Toast {
@@ -322,7 +339,8 @@ impl Backend {
/// Fetches the OmniGen model listing for `modality`.
pub fn load_models(&self, modality: Modality) {
let host = self.host.clone();
let api = self.api();
let host = api.host.clone();
self.spawn(move |tx| async move {
let mut last_err = String::new();
for attempt in 0..MODEL_LIST_ATTEMPTS {
@@ -330,15 +348,16 @@ impl Backend {
tokio::time::sleep(MODEL_LIST_BACKOFF).await;
}
let result = match modality {
Modality::Image => omni_gen_list_image_models(OmniGenListImageModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::image_model).collect::<Vec<_>>()),
Modality::Video => omni_gen_list_video_models(OmniGenListVideoModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::video_model).collect::<Vec<_>>()),
Modality::Image => omni_gen_list_image_models(OmniGenListImageModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::image_model).collect::<Vec<_>>()).map_err(|e| api_message(&e)),
Modality::Video => omni_gen_list_video_models(OmniGenListVideoModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::video_model).collect::<Vec<_>>()).map_err(|e| api_message(&e)),
Modality::Audio => api.get_json::<AudioModels>("/v1/omni_gen/models/audio").await.map(|r| r.models.iter().map(models::audio_model).collect::<Vec<_>>()),
};
match result {
Ok(models) => {
let _ = tx.send(Event::Models(modality, Ok(models)));
return;
},
Err(err) => last_err = api_message(&err),
Err(err) => last_err = err,
}
}
error!("Listing {modality:?} models failed: {last_err}");
@@ -367,7 +386,7 @@ impl Backend {
let api = self.api();
self.spawn(move |tx| async move {
let Some(creds) = api.creds.as_ref() else {
let _ = tx.send(Event::EnqueueFailed { modality, message: "Please sign in to generate.".to_owned() });
let _ = tx.send(Event::EnqueueFailed { modality, message: "Please sign in to generate.".to_owned(), status: None });
return;
};
fields.entry("idempotency_token").or_insert_with(|| Value::String(uuid::Uuid::new_v4().to_string()));
@@ -376,14 +395,14 @@ impl Backend {
Ok(response) => {
let tokens = job_tokens(&response);
if tokens.is_empty() {
let _ = tx.send(Event::EnqueueFailed { modality, message: "The server didn't start a job.".to_owned() });
let _ = tx.send(Event::EnqueueFailed { modality, message: "The server didn't start a job.".to_owned(), status: None });
} else {
let _ = tx.send(Event::Enqueued { modality, job_tokens: tokens, meta });
}
api.send_credits(&tx).await;
},
Err(err) => {
let _ = tx.send(Event::EnqueueFailed { modality, message: api_message(&err) });
let _ = tx.send(Event::EnqueueFailed { modality, message: api_message(&err), status: api_status(&err) });
},
}
});
@@ -485,6 +504,92 @@ impl Backend {
});
}
/// Loads every page of the user's characters.
pub fn load_characters(&self) {
let api = self.api();
if api.creds.is_none() {
return;
}
self.spawn(move |tx| async move {
let mut all = Vec::new();
let mut cursor: Option<i64> = None;
loop {
let path = match cursor {
Some(c) => format!("/v1/characters/session?cursor={c}"),
None => "/v1/characters/session".to_owned(),
};
match api.request_json::<CharactersPage>(reqwest::Method::GET, &path, None).await {
Ok(page) => {
all.extend(page.characters);
match page.next_cursor {
Some(next) if Some(next) != cursor => cursor = Some(next),
_ => break,
}
},
Err(err) => {
warn!("Loading characters failed: {err}");
break;
},
}
}
let _ = tx.send(Event::Characters(all));
});
}
/// Starts making a character from an uploaded reference image (`POST /v1/character/create`).
pub fn create_character(&self, image_token: String, name: String, description: Option<String>) {
let api = self.api();
self.spawn(move |tx| async move {
let body = serde_json::json!({ "image_media_token": image_token, "model": CHARACTER_MODEL, "uuid_idempotency_token": uuid::Uuid::new_v4().to_string(), "character_name": name, "character_description": description });
match api.request_json::<Value>(reqwest::Method::POST, "/v1/character/create", Some(body)).await {
Ok(r) => match r.get("inference_job_token").and_then(Value::as_str) {
Some(job) => {
let _ = tx.send(Event::CharacterCreating { job_token: job.to_owned(), name });
},
None => {
let _ = tx.send(Event::Toast { error: true, message: "Failed to create character".to_owned() });
},
},
Err(err) => {
let _ = tx.send(Event::Toast { error: true, message: if err.is_empty() { "Failed to create character".to_owned() } else { err } });
},
}
});
}
/// Renames or re-describes a character (`POST /v1/character/edit`).
pub fn edit_character(&self, token: String, name: String, description: String) {
let api = self.api();
self.spawn(move |tx| async move {
let body = serde_json::json!({ "token": token, "updated_name": name, "updated_description": description });
match api.request_json::<Value>(reqwest::Method::POST, "/v1/character/edit", Some(body)).await {
Ok(_) => {
let _ = tx.send(Event::Toast { error: false, message: "Character updated".to_owned() });
let _ = tx.send(Event::CharacterChanged);
},
Err(err) => {
let _ = tx.send(Event::Toast { error: true, message: format!("Failed to update character: {err}") });
},
}
});
}
/// `DELETE /v1/character/{token}`.
pub fn delete_character(&self, token: String, name: String) {
let api = self.api();
self.spawn(move |tx| async move {
match api.request_json::<Value>(reqwest::Method::DELETE, &format!("/v1/character/{token}"), None).await {
Ok(_) => {
let _ = tx.send(Event::Toast { error: false, message: format!("Character \"{name}\" deleted") });
let _ = tx.send(Event::CharacterChanged);
},
Err(err) => {
let _ = tx.send(Event::Toast { error: true, message: format!("Failed to delete character: {err}") });
},
}
});
}
pub fn delete_media(&self, token: String) {
let api = self.api();
self.spawn(move |tx| async move {
@@ -589,8 +694,16 @@ struct Api {
impl Api {
/// A GET with the desktop app's identity headers and cookies, decoded leniently.
async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, String> {
self.request_json(reqwest::Method::GET, path, None).await
}
/// Any JSON call with the desktop app's identity headers and cookies, decoded leniently.
async fn request_json<T: DeserializeOwned>(&self, method: reqwest::Method, path: &str, body: Option<Value>) -> Result<T, String> {
let url = format!("{}{path}", self.host.to_api_hostname_and_scheme());
let mut request = self.http.get(&url).header("Accept", "application/json").header("Origin", self.host.request_origin());
let mut request = self.http.request(method, &url).header("Accept", "application/json").header("Origin", self.host.request_origin());
if let Some(body) = body {
request = request.json(&body);
}
if let Some(cookie) = self.creds.as_ref().and_then(StorytellerCredentialSet::maybe_as_cookie_header) {
request = request.header("Cookie", cookie);
}
@@ -644,7 +757,7 @@ impl Api {
}
let info = self.get_json::<FileResponse>(&format!("/v1/media_files/file/{token}")).await.ok();
let file = info.map(|r| r.media_file);
UploadedMedia { duration_secs: file.as_ref().and_then(|f| f.maybe_duration_millis).map_or(0.0, |ms| ms as f32 / 1000.0), thumbnail: file.as_ref().and_then(|f| f.media_links.thumbnail(256)), full_url: file.as_ref().map(|f| f.media_links.cdn_url.clone()).filter(|u| !u.is_empty()), token }
UploadedMedia { duration_secs: file.as_ref().and_then(|f| f.maybe_duration_millis).map_or(0.0, |ms| ms as f32 / 1000.0), thumbnail: file.as_ref().and_then(|f| f.thumbnail(256)), full_url: file.as_ref().map(|f| f.media_links.cdn_url.clone()).filter(|u| !u.is_empty()), token }
}
}
@@ -687,6 +800,24 @@ pub fn api_message(err: &StorytellerError) -> String {
}
}
/// The HTTP status behind a client error, when there was a response.
fn api_status(err: &StorytellerError) -> Option<u16> {
let StorytellerError::Api(api) = err else {
return None;
};
Some(match api {
ApiError::InvalidRequest(_) => 400,
ApiError::Unauthorized(_) => 401,
ApiError::PaymentRequired(_) => 402,
ApiError::Forbidden(_) => 403,
ApiError::NotFound(_) => 404,
ApiError::TooManyRequests(_) => 429,
ApiError::InternalServerError { .. } => 500,
ApiError::UncategorizedBadResponseWithStatusAndBody { status_code, .. } => status_code.as_u16(),
_ => return None,
})
}
fn api_error_message(err: &ApiError) -> String {
let body = match err {
ApiError::InvalidRequest(b) | ApiError::Unauthorized(b) | ApiError::PaymentRequired(b) | ApiError::Forbidden(b) | ApiError::NotFound(b) | ApiError::TooManyRequests(b) => Some(b.as_str()),
+422
View File
@@ -0,0 +1,422 @@
//! In-app video playback through FFmpeg (when it's installed): `ffprobe` reads the clip's size,
//! rate and length, one `ffmpeg` streams RGBA frames into a texture and another streams PCM into
//! rodio. Pausing and seeking restart both at the new position. Without FFmpeg the lightbox opens
//! videos in the system player instead.
use std::collections::VecDeque;
use std::io::Read;
use std::process::{Child, Command, Stdio};
use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, channel, sync_channel};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use egui::{Align2, Color32, ColorImage, FontId, Rect, Sense, TextureHandle, TextureOptions, Ui, pos2, vec2};
use log::warn;
use rodio::{OutputStream, OutputStreamHandle, Sink, Source};
use crate::theme;
use crate::ui::icons::{self, Icon};
/// Frames are scaled down to at most this width (the lightbox never shows them larger).
const MAX_WIDTH: u32 = 1280;
const AUDIO_RATE: u32 = 44_100;
/// Decoded frames buffered ahead of the clock (back-pressure on `ffmpeg`).
const FRAME_QUEUE: usize = 4;
const CONTROLS_HEIGHT: f32 = 40.0;
/// Whether `ffmpeg` and `ffprobe` are on the PATH (checked once).
pub fn ffmpeg_available() -> bool {
static AVAILABLE: OnceLock<bool> = OnceLock::new();
*AVAILABLE.get_or_init(|| ["ffmpeg", "ffprobe"].iter().all(|tool| command(tool).arg("-version").stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success())))
}
#[derive(Clone, Copy, Debug)]
struct VideoInfo {
width: u32,
height: u32,
fps: f64,
duration: f64,
}
pub struct VideoPlayer {
url: String,
probe: Option<Receiver<Result<VideoInfo, String>>>,
info: Option<VideoInfo>,
stream: Option<Stream>,
texture: Option<TextureHandle>,
/// Media time of the frame on screen.
position: f64,
muted: bool,
error: Option<String>,
output: Option<(OutputStream, OutputStreamHandle)>,
}
/// One run of the decoders from a start position.
struct Stream {
start: f64,
started: Instant,
frames: Receiver<(ColorImage, f64)>,
next: Option<(ColorImage, f64)>,
ended: bool,
children: Vec<Child>,
sink: Option<Sink>,
}
impl Drop for Stream {
fn drop(&mut self) {
for child in &mut self.children {
let _ = child.kill();
let _ = child.wait();
}
}
}
impl VideoPlayer {
/// Starts probing `url`; playback begins as soon as its size is known.
pub fn new(url: &str) -> Self {
let (tx, rx) = channel();
let probe_url = url.to_owned();
std::thread::spawn(move || {
let _ = tx.send(probe(&probe_url));
});
Self { url: url.to_owned(), probe: Some(rx), info: None, stream: None, texture: None, position: 0.0, muted: false, error: None, output: None }
}
pub fn url(&self) -> &str {
&self.url
}
/// Draws the video fitted inside `rect` with its controls; `poster` (the still frame) shows
/// until the first decoded frame arrives.
pub fn ui(&mut self, ui: &mut Ui, rect: Rect, poster: Option<&TextureHandle>) {
self.poll_probe();
let ctx = ui.ctx().clone();
self.advance(&ctx);
ui.painter().rect_filled(rect, 0.0, Color32::BLACK);
let video_area = Rect::from_min_max(rect.min, pos2(rect.right(), rect.bottom() - CONTROLS_HEIGHT));
match (&self.texture, &self.error) {
(_, Some(err)) => {
ui.painter().text(video_area.center(), Align2::CENTER_CENTER, format!("Couldn't play this video: {err}"), FontId::new(13.0, egui::FontFamily::Proportional), theme::BAD);
},
(Some(texture), None) => {
let fit = crate::overlays::lightbox::fit_inside(texture.size_vec2(), video_area.shrink(4.0).size());
egui::Image::new(texture).paint_at(ui, Rect::from_center_size(video_area.center(), fit));
},
(None, None) => {
if let Some(poster) = poster {
let fit = crate::overlays::lightbox::fit_inside(poster.size_vec2(), video_area.shrink(4.0).size());
egui::Image::new(poster).tint(Color32::from_gray(150)).paint_at(ui, Rect::from_center_size(video_area.center(), fit));
}
icons::paint_spinner(ui, Rect::from_center_size(video_area.center(), vec2(32.0, 32.0)), Color32::WHITE);
},
}
// Clicking the picture toggles play/pause.
let click = ui.interact(video_area, ui.id().with(("video-surface", &self.url)), Sense::click());
if click.clicked() {
self.toggle_pause();
}
self.controls(ui, Rect::from_min_max(pos2(rect.left(), rect.bottom() - CONTROLS_HEIGHT), rect.max));
if self.stream.is_some() {
ctx.request_repaint_after(Duration::from_millis(10));
}
}
fn controls(&mut self, ui: &mut Ui, bar: Rect) {
let p = ui.painter().clone();
p.rect_filled(bar, 0.0, theme::fade(theme::PANEL, 0.95));
let playing = self.stream.is_some();
let button = Rect::from_center_size(pos2(bar.left() + 22.0, bar.center().y), vec2(28.0, 28.0));
let resp = ui.interact(button, ui.id().with(("video-play", &self.url)), Sense::click()).on_hover_text(if playing { "Pause" } else { "Play" }).on_hover_cursor(egui::CursorIcon::PointingHand);
if playing {
for dx in [-3.5, 3.5] {
p.rect_filled(Rect::from_center_size(button.center() + vec2(dx, 0.0), vec2(3.0, 12.0)), 0.0, theme::INK);
}
} else {
icons::paint(&p, button.shrink(7.0), Icon::Play, theme::INK);
}
if resp.clicked() {
self.toggle_pause();
}
let duration = self.info.map_or(0.0, |i| i.duration);
let time = format!("{} / {}", clock(self.position), clock(duration));
let label = p.text(pos2(button.right() + 8.0, bar.center().y), Align2::LEFT_CENTER, time, FontId::new(11.5, egui::FontFamily::Monospace), theme::MUTED);
let mute = Rect::from_center_size(pos2(bar.right() - 22.0, bar.center().y), vec2(28.0, 28.0));
let mute_resp = ui.interact(mute, ui.id().with(("video-mute", &self.url)), Sense::click()).on_hover_text(if self.muted { "Unmute" } else { "Mute" }).on_hover_cursor(egui::CursorIcon::PointingHand);
icons::paint(&p, mute.shrink(7.0), if self.muted { Icon::VolumeOff } else { Icon::Volume }, theme::INK);
if mute_resp.clicked() {
self.muted = !self.muted;
if let Some(sink) = self.stream.as_ref().and_then(|s| s.sink.as_ref()) {
sink.set_volume(if self.muted { 0.0 } else { 1.0 });
}
}
// The scrubber: click or drag to seek.
let track = Rect::from_min_max(pos2(label.right() + 14.0, bar.center().y - 8.0), pos2(mute.left() - 12.0, bar.center().y + 8.0));
let scrub = ui.interact(track, ui.id().with(("video-scrub", &self.url)), Sense::click_and_drag()).on_hover_cursor(egui::CursorIcon::PointingHand);
let line = Rect::from_center_size(track.center(), vec2(track.width(), 4.0));
p.rect_filled(line, egui::CornerRadius::same(2), theme::WASH_HOVER);
let fraction = if duration > 0.0 { (self.position / duration).clamp(0.0, 1.0) as f32 } else { 0.0 };
p.rect_filled(Rect::from_min_size(line.min, vec2(line.width() * fraction, line.height())), egui::CornerRadius::same(2), theme::ACCENT_400);
if (scrub.clicked() || scrub.drag_stopped()) && duration > 0.0 {
if let Some(pos) = scrub.interact_pointer_pos() {
let t = (((pos.x - line.left()) / line.width()).clamp(0.0, 1.0) as f64) * duration;
self.seek(t);
}
}
}
fn poll_probe(&mut self) {
let Some(rx) = &self.probe else {
return;
};
match rx.try_recv() {
Ok(Ok(info)) => {
self.info = Some(info);
self.probe = None;
self.play_from(0.0);
},
Ok(Err(err)) => {
self.error = Some(err);
self.probe = None;
},
Err(_) => {},
}
}
/// Shows the newest frame that's due; loops at the end.
fn advance(&mut self, ctx: &egui::Context) {
let Some(stream) = &mut self.stream else {
return;
};
let now = stream.start + stream.started.elapsed().as_secs_f64();
let mut shown = None;
loop {
if stream.next.is_none() {
match stream.frames.try_recv() {
Ok(frame) => stream.next = Some(frame),
Err(TryRecvError::Disconnected) => {
stream.ended = true;
break;
},
Err(TryRecvError::Empty) => break,
}
}
match &stream.next {
Some((_, pts)) if *pts <= now => shown = stream.next.take(),
_ => break,
}
}
if let Some((image, pts)) = shown {
self.position = pts;
match &mut self.texture {
Some(t) => t.set(image, TextureOptions::LINEAR),
None => self.texture = Some(ctx.load_texture(format!("video:{}", self.url), image, TextureOptions::LINEAR)),
}
}
if stream.ended && stream.next.is_none() {
self.play_from(0.0);
}
}
fn toggle_pause(&mut self) {
if self.stream.take().is_none() && self.info.is_some() {
self.play_from(self.position);
}
}
fn seek(&mut self, t: f64) {
self.position = t;
self.play_from(t);
}
fn play_from(&mut self, start: f64) {
self.stream = None;
let Some(info) = self.info else {
return;
};
match self.spawn(info, start) {
Ok(stream) => self.stream = Some(stream),
Err(err) => self.error = Some(err),
}
}
fn spawn(&mut self, info: VideoInfo, start: f64) -> Result<Stream, String> {
let (w, h) = output_size(info);
let mut video = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{start:.3}"), "-i", &self.url, "-an", "-vf", &format!("scale={w}:{h}"), "-f", "rawvideo", "-pix_fmt", "rgba", "-"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(|e| format!("ffmpeg: {e}"))?;
let stdout = video.stdout.take().ok_or("ffmpeg has no output")?;
let (frame_tx, frame_rx) = sync_channel(FRAME_QUEUE);
let fps = info.fps.max(1.0);
std::thread::spawn(move || read_frames(stdout, w, h, start, fps, frame_tx));
let mut children = vec![video];
let sink = self.spawn_audio(start, &mut children);
Ok(Stream { start, started: Instant::now(), frames: frame_rx, next: None, ended: false, children, sink })
}
/// The soundtrack as PCM into rodio (silently skipped when the clip has none).
fn spawn_audio(&mut self, start: f64, children: &mut Vec<Child>) -> Option<Sink> {
let mut audio = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{start:.3}"), "-i", &self.url, "-vn", "-ac", "2", "-ar", &AUDIO_RATE.to_string(), "-f", "s16le", "-"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().ok()?;
let mut stdout = audio.stdout.take()?;
children.push(audio);
if self.output.is_none() {
self.output = OutputStream::try_default().map_err(|e| warn!("No audio output: {e}")).ok();
}
let (_, handle) = self.output.as_ref()?;
let sink = Sink::try_new(handle).ok()?;
let (tx, rx) = channel::<Vec<i16>>();
std::thread::spawn(move || {
let mut buf = vec![0u8; 8192];
while let Ok(n) = stdout.read(&mut buf) {
if n == 0 || tx.send(buf[..n - n % 2].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()).is_err() {
break;
}
}
});
sink.set_volume(if self.muted { 0.0 } else { 1.0 });
sink.append(PcmStream { rx, buf: VecDeque::new() });
Some(sink)
}
}
/// Reads fixed-size RGBA frames from `ffmpeg` and stamps them with their media time.
fn read_frames(mut stdout: impl Read, w: u32, h: u32, start: f64, fps: f64, tx: SyncSender<(ColorImage, f64)>) {
let mut buf = vec![0u8; (w * h * 4) as usize];
let mut n = 0u64;
while stdout.read_exact(&mut buf).is_ok() {
let image = ColorImage::from_rgba_unmultiplied([w as usize, h as usize], &buf);
if tx.send((image, start + n as f64 / fps)).is_err() {
break;
}
n += 1;
}
}
/// Live PCM from a channel; silence while it waits, the end when the sender's gone.
struct PcmStream {
rx: Receiver<Vec<i16>>,
buf: VecDeque<i16>,
}
impl Iterator for PcmStream {
type Item = i16;
fn next(&mut self) -> Option<i16> {
if self.buf.is_empty() {
match self.rx.try_recv() {
Ok(chunk) => self.buf.extend(chunk),
Err(TryRecvError::Empty) => return Some(0),
Err(TryRecvError::Disconnected) => return None,
}
}
self.buf.pop_front().or(Some(0))
}
}
impl Source for PcmStream {
fn current_frame_len(&self) -> Option<usize> {
None
}
fn channels(&self) -> u16 {
2
}
fn sample_rate(&self) -> u32 {
AUDIO_RATE
}
fn total_duration(&self) -> Option<Duration> {
None
}
}
/// Reads the clip's size, rate and length. A small probe keeps this ~2 s on CDN files instead of
/// ~14 s (ffprobe otherwise scans much of the file over HTTP).
fn probe(url: &str) -> Result<VideoInfo, String> {
#[derive(serde::Deserialize, Default)]
#[serde(default)]
struct Probe {
streams: Vec<ProbeStream>,
format: ProbeFormat,
}
#[derive(serde::Deserialize, Default)]
#[serde(default)]
struct ProbeStream {
width: u32,
height: u32,
avg_frame_rate: String,
}
#[derive(serde::Deserialize, Default)]
#[serde(default)]
struct ProbeFormat {
duration: String,
}
let out = command("ffprobe").args(["-v", "error", "-probesize", "65536", "-analyzeduration", "0", "-select_streams", "v:0", "-show_entries", "stream=width,height,avg_frame_rate:format=duration", "-of", "json", url]).output().map_err(|e| format!("ffprobe: {e}"))?;
let probe: Probe = serde_json::from_slice(&out.stdout).map_err(|e| format!("ffprobe: {e}"))?;
let stream = probe.streams.first().ok_or("no video stream")?;
let fps = match stream.avg_frame_rate.split_once('/') {
Some((n, d)) => n.parse::<f64>().unwrap_or(24.0) / d.parse::<f64>().unwrap_or(1.0).max(1.0),
None => stream.avg_frame_rate.parse().unwrap_or(24.0),
};
Ok(VideoInfo { width: stream.width.max(2), height: stream.height.max(2), fps: if fps.is_finite() && fps > 0.0 { fps } else { 24.0 }, duration: probe.format.duration.parse().unwrap_or(0.0) })
}
/// The decoded size: at most `MAX_WIDTH` wide, both sides even (as `scale` requires).
fn output_size(info: VideoInfo) -> (u32, u32) {
let w = info.width.min(MAX_WIDTH) & !1;
let h = ((info.height as f64 * w as f64 / info.width as f64).round() as u32).max(2) & !1;
(w, h)
}
/// `m:ss`.
fn clock(secs: f64) -> String {
let s = secs.max(0.0) as u64;
format!("{}:{:02}", s / 60, s % 60)
}
/// A command that doesn't flash a console window on Windows.
fn command(program: &str) -> Command {
#[allow(unused_mut)]
let mut cmd = Command::new(program);
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
cmd.creation_flags(CREATE_NO_WINDOW);
}
cmd
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn output_sizes_are_even_and_capped() {
let info = |width, height| VideoInfo { width, height, fps: 24.0, duration: 5.0 };
assert_eq!(output_size(info(1920, 1080)), (1280, 720));
assert_eq!(output_size(info(721, 1281)), (720, 1278));
assert_eq!(clock(65.9), "1:05");
}
/// Needs FFmpeg on the PATH: `cargo test -p artcraft_native -- --ignored`.
#[test]
#[ignore]
fn probes_and_decodes_a_generated_clip() {
let dir = std::env::temp_dir().join("artcraft-native-video-test");
std::fs::create_dir_all(&dir).unwrap();
let clip = dir.join("clip.mp4");
let status = command("ffmpeg").args(["-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc=duration=2:size=640x360:rate=24", "-f", "lavfi", "-i", "sine=duration=2", "-shortest", "-pix_fmt", "yuv420p"]).arg(&clip).status().unwrap();
assert!(status.success());
let url = clip.to_string_lossy().into_owned();
let info = probe(&url).unwrap();
assert_eq!((info.width, info.height), (640, 360));
assert!((info.fps - 24.0).abs() < 0.01 && (info.duration - 2.0).abs() < 0.2);
let mut player = VideoPlayer { url: url.clone(), probe: None, info: Some(info), stream: None, texture: None, position: 0.0, muted: true, error: None, output: None };
let stream = player.spawn(info, 1.0).unwrap();
let (frame, pts) = stream.frames.recv_timeout(Duration::from_secs(10)).unwrap();
assert_eq!(frame.size, [640, 360]);
assert!((pts - 1.0).abs() < 1e-9, "frames are stamped from the start position");
let count = 1 + stream.frames.iter().count();
assert!((20..=28).contains(&count), "about a second of frames at 24 fps, got {count}");
}
}
+87
View File
@@ -42,6 +42,17 @@ impl MediaLinks {
}
}
impl MediaLinks {
/// A video's animated preview, `width` pixels wide where the CDN can resize.
pub fn animated_preview(&self, width: u32) -> Option<String> {
let previews = self.maybe_video_previews.as_ref()?;
if !previews.animated_thumbnail_template.is_empty() {
return Some(previews.animated_thumbnail_template.replace("{WIDTH}", &width.to_string()));
}
(!previews.animated.is_empty()).then(|| previews.animated.clone())
}
}
/// `GET /v1/media_files/list/user/{username}`.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
@@ -70,10 +81,56 @@ pub struct MediaFile {
pub maybe_model_type: Option<String>,
pub maybe_origin_model_type: Option<String>,
pub maybe_duration_millis: Option<u64>,
pub maybe_title: Option<String>,
pub media_links: MediaLinks,
pub created_at: String,
}
impl MediaFile {
/// The file's still thumbnail; audio files have none (their CDN link is the audio itself).
pub fn thumbnail(&self, width: u32) -> Option<String> {
if self.media_class.as_deref() == Some("audio") {
return None;
}
self.media_links.thumbnail(width)
}
}
/// `GET /v1/omni_gen/models/audio`. The client has no binding for it (and the API's type only
/// serializes), so it's read here; capability flags are left out when false.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
pub struct AudioModels {
pub success: bool,
pub models: Vec<AudioModel>,
}
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(default)]
pub struct AudioModel {
pub model: String,
pub model_creator: Option<String>,
pub full_name: Option<String>,
pub extra_info_short: Option<String>,
pub text_prompt_supported: Option<bool>,
pub style_prompt_supported: bool,
pub audio_references_supported: bool,
pub audio_references_max: Option<u16>,
pub image_references_supported: bool,
pub image_references_max: Option<u16>,
pub keep_lyrics_supported: bool,
pub instrumental_toggle_supported: bool,
pub loopable_toggle_supported: bool,
pub bpm_supported: bool,
pub musical_key_supported: bool,
pub sample_rate_hz_options: Vec<u32>,
pub sample_rate_hz_default: Option<u32>,
pub speed_supported: bool,
pub volume_supported: bool,
pub pitch_supported: bool,
pub is_disabled: bool,
}
/// `GET /v1/media_files/batch_gen_redux/{token}`.
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
@@ -194,6 +251,36 @@ pub struct ContextImage {
pub media_links: MediaLinks,
}
/// A saved character (`GET /v1/characters/session`).
#[derive(Clone, Debug, Default, Deserialize)]
#[serde(default)]
pub struct Character {
pub token: String,
pub name: String,
pub maybe_description: Option<String>,
pub maybe_avatar: Option<MediaLinks>,
pub maybe_full_image: Option<MediaLinks>,
pub models: Vec<String>,
}
impl Character {
pub fn avatar_url(&self) -> Option<String> {
self.maybe_avatar.as_ref().or(self.maybe_full_image.as_ref()).and_then(|l| l.thumbnail(256))
}
pub fn full_url(&self) -> Option<String> {
self.maybe_full_image.as_ref().or(self.maybe_avatar.as_ref()).map(|l| l.cdn_url.clone()).filter(|u| !u.is_empty())
}
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
pub struct CharactersPage {
pub success: bool,
pub characters: Vec<Character>,
pub next_cursor: Option<i64>,
}
/// `GET /v1/session` (only the parts the app shows).
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
+128 -57
View File
@@ -1,16 +1,17 @@
//! The generation feed as a masonry grid (`GenerationGridView`) or a list (`GenerationListView`):
//! running jobs first-class alongside failures and finished media, newest first, loading more as
//! the user scrolls.
//! the user scrolls. Audio plays in place, with its waveform.
use std::collections::{HashMap, HashSet};
use egui::{Align2, Color32, CornerRadius, FontFamily, FontId, Id, Mesh, Rect, Sense, Stroke, Ui, pos2, vec2};
use crate::backend::audio::AudioPlayer;
use crate::backend::media_cache::{Lookup, MediaCache};
use crate::feed::types::{FailedJob, FeedItem, MediaKind, PendingJob, pending_status, time_ago};
use crate::theme;
use crate::ui::icons::{self, Icon};
use crate::ui::{creator_icons, widgets};
use crate::ui::{creator_icons, waveform, widgets};
/// `react-masonry-css` gutter and the cap on tall portraits (≈ 5:7).
const GAP: f32 = 8.0;
@@ -18,6 +19,8 @@ const MAX_RATIO: f32 = 1.4;
/// Start loading the next page this far before the end.
const LOAD_MORE_MARGIN: f32 = 400.0;
const LIST_THUMB: f32 = 100.0;
/// The list's compact audio player is at most `max-w-md` wide.
const LIST_PLAYER_MAX: f32 = 448.0;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum ViewMode {
@@ -61,6 +64,8 @@ pub struct FeedView<'a> {
/// Prompt texts by prompt token (list view and pending cards).
pub prompts: &'a HashMap<String, String>,
pub make_video: bool,
/// Play videos' animated previews instead of their still frames.
pub autoplay: bool,
pub bottom_padding: f32,
}
@@ -84,7 +89,7 @@ impl Entry<'_> {
#[derive(Clone, Default)]
pub struct RatioCache(HashMap<String, f32>);
pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &mut RatioCache, names: &dyn ModelNames) -> Vec<FeedAction> {
pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &mut RatioCache, audio: &mut AudioPlayer, names: &dyn ModelNames) -> Vec<FeedAction> {
let mut entries: Vec<Entry<'_>> = view.pending.iter().map(Entry::Pending).chain(view.failed.iter().map(Entry::Failed)).chain(view.items.iter().map(Entry::Item)).collect();
entries.sort_by_key(|e| std::cmp::Reverse(e.time()));
let mut actions = Vec::new();
@@ -111,10 +116,10 @@ pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &m
match (entry, view.mode) {
(Entry::Pending(job), ViewMode::Grid) => pending_card(ui, rect, job, now, names),
(Entry::Failed(job), ViewMode::Grid) => failed_card(ui, view.id, rect, job, cache, names, &mut actions),
(Entry::Item(item), ViewMode::Grid) => item_card(ui, view, rect, item, cache, ratios, names, &mut actions),
(Entry::Item(item), ViewMode::Grid) => item_card(ui, view, rect, item, cache, ratios, audio, names, &mut actions),
(Entry::Pending(job), ViewMode::List) => pending_row(ui, rect, job, now, names),
(Entry::Failed(job), ViewMode::List) => failed_row(ui, view.id, rect, job, names, &mut actions),
(Entry::Item(item), ViewMode::List) => item_row(ui, view, rect, item, cache, names, now, &mut actions),
(Entry::Item(item), ViewMode::List) => item_row(ui, view, rect, item, cache, audio, names, now, &mut actions),
}
}
@@ -226,25 +231,15 @@ fn failed_card(ui: &mut Ui, id: Id, rect: Rect, job: &FailedJob, cache: &mut Med
}
#[allow(clippy::too_many_arguments)]
fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache, names: &dyn ModelNames, actions: &mut Vec<FeedAction>) {
fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache, audio: &mut AudioPlayer, names: &dyn ModelNames, actions: &mut Vec<FeedAction>) {
let resp = ui.interact(rect, view.id.with(("item", &item.token)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
let hovered = resp.hovered();
ui.painter().rect_filled(rect, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4));
match item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
Some(Lookup::Ready(t)) => {
let size = t.size_vec2();
if size.x > 0.0 && !ratios.0.contains_key(&item.token) {
ratios.0.insert(item.token.clone(), size.y / size.x);
}
egui::Image::new(&t).uv(cover_uv_for(size, rect.size())).corner_radius(theme::RADIUS).paint_at(ui, rect);
},
Some(Lookup::Loading) => shimmer(ui, rect),
_ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), if item.kind == MediaKind::Video { Icon::Video } else { Icon::Image }, theme::FAINT),
}
if item.kind == MediaKind::Video {
let badge = Rect::from_center_size(rect.center(), vec2(40.0, 40.0));
ui.painter().circle_filled(badge.center(), 20.0, Color32::from_black_alpha(110));
icons::paint(ui.painter(), badge.shrink(12.0).translate(vec2(1.5, 0.0)), Icon::Play, theme::fade(Color32::WHITE, 0.9));
// The pointer may be on an action button (or the audio player) rather than the card itself.
let hovered = resp.hovered() || ui.rect_contains_pointer(rect);
let is_audio = item.kind == MediaKind::Audio;
if is_audio {
audio_face(ui, view, rect, item, audio);
} else {
visual_face(ui, view, rect, item, cache, ratios);
}
let selected = view.selected.contains(&item.token);
let ring = if selected {
@@ -276,25 +271,17 @@ fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cach
p.galley(pos2(badge.left() + 20.0, badge.center().y - galley.size().y / 2.0), galley, Color32::WHITE);
}
let mut action_clicked = false;
// The pointer may be on an action button rather than the card itself.
if (hovered || ui.rect_contains_pointer(rect)) && !view.selecting {
let footer = bottom_gradient(ui, rect, 56.0, 178);
let chip_y = footer.bottom() - 18.0;
let mut x = footer.left() + 8.0;
x = chip(ui, pos2(x, chip_y), Some(if item.kind == MediaKind::Video { Icon::Video } else { Icon::Image }), None, item.kind.label());
if hovered && !view.selecting {
// Audio keeps its player at the bottom, so its chips and actions go on top.
let band = edge_fade(ui, rect, 56.0, 178, is_audio);
let chip_y = if is_audio { band.top() + 18.0 } else { band.bottom() - 18.0 };
let mut x = band.left() + 8.0;
x = chip(ui, pos2(x, chip_y), Some(kind_icon(item.kind)), None, item.kind.label());
if let Some(model) = item.model_id.as_deref() {
chip(ui, pos2(x + 6.0, chip_y), None, Some(&names.creator(model)), &truncate(&names.display_name(model), 16));
}
let mut buttons: Vec<(Icon, &str, FeedAction)> = Vec::new();
if item.prompt_token.is_some() {
buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone())));
}
if view.make_video && item.kind == MediaKind::Image {
buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone())));
}
buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone())));
buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone())));
let pill = Rect::from_min_size(pos2(footer.right() - 8.0 - 8.0 - 28.0 * buttons.len() as f32, chip_y - 18.0), vec2(8.0 + 28.0 * buttons.len() as f32, 36.0));
let buttons = item_actions(view, item, None);
let pill = Rect::from_min_size(pos2(band.right() - 8.0 - 8.0 - 28.0 * buttons.len() as f32, chip_y - 18.0), vec2(8.0 + 28.0 * buttons.len() as f32, 36.0));
ui.painter().rect_filled(pill, 0.0, Color32::from_black_alpha(153));
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(pill.shrink(4.0)).layout(egui::Layout::left_to_right(egui::Align::Center)));
child.spacing_mut().item_spacing.x = 0.0;
@@ -310,6 +297,87 @@ fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cach
}
}
/// An image or video card's face: the thumbnail (or animated preview) and a video's play badge.
fn visual_face(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache) {
ui.painter().rect_filled(rect, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4));
// Videos play their animated preview when previews are on (the still shows while it loads).
let animated = item.animated.as_deref().filter(|_| view.autoplay && item.kind == MediaKind::Video).map(|u| cache.get_animated(ui.ctx(), u));
let lookup = match animated {
Some(Lookup::Ready(t)) => Some(Lookup::Ready(t)),
_ => item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)),
};
match lookup {
Some(Lookup::Ready(t)) => {
let size = t.size_vec2();
if size.x > 0.0 && !ratios.0.contains_key(&item.token) {
ratios.0.insert(item.token.clone(), size.y / size.x);
}
egui::Image::new(&t).uv(cover_uv_for(size, rect.size())).corner_radius(theme::RADIUS).paint_at(ui, rect);
},
Some(Lookup::Loading) => shimmer(ui, rect),
_ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), kind_icon(item.kind), theme::FAINT),
}
if item.kind == MediaKind::Video && !(view.autoplay && item.animated.is_some()) {
let badge = Rect::from_center_size(rect.center(), vec2(40.0, 40.0));
ui.painter().circle_filled(badge.center(), 20.0, Color32::from_black_alpha(110));
icons::paint(ui.painter(), badge.shrink(12.0).translate(vec2(1.5, 0.0)), Icon::Play, theme::fade(Color32::WHITE, 0.9));
}
}
/// An audio card's face (`GalleryCard`'s audio branch, a square tile): the title, a music note
/// and the waveform player along the bottom.
fn audio_face(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, audio: &mut AudioPlayer) {
let mut mesh = Mesh::default();
let (top, bottom) = (theme::fade(theme::ACCENT, 0.14), theme::fade(theme::CONTROLS, 0.6));
mesh.colored_vertex(rect.left_top(), top);
mesh.colored_vertex(rect.right_top(), top);
mesh.colored_vertex(rect.right_bottom(), bottom);
mesh.colored_vertex(rect.left_bottom(), bottom);
mesh.add_triangle(0, 1, 2);
mesh.add_triangle(0, 2, 3);
ui.painter().add(mesh);
// Clear of the selection checkbox when selecting.
let inset = if view.selecting { 36.0 } else { 12.0 };
let title = item.prompt_token.as_deref().and_then(|t| view.prompts.get(t)).cloned().unwrap_or_else(|| fallback_title(item));
let galley = clamp_lines(ui, &title, 13.0, theme::fade(Color32::WHITE, 0.85), rect.width() - inset - 12.0, 2);
ui.painter().galley(pos2(rect.left() + inset, rect.top() + 12.0), galley, Color32::WHITE);
waveform::music_tile(ui, Rect::from_center_size(rect.center() - vec2(0.0, 12.0), vec2(48.0, 48.0)), true);
let bar = Rect::from_min_max(pos2(rect.left() + 12.0, rect.bottom() - 56.0), pos2(rect.right() - 12.0, rect.bottom() - 12.0));
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar));
waveform::player(&mut child, view.id.with(("player", &item.token)), audio, &item.full_url, item.duration_secs, false);
}
/// The hover actions for an item: copy prompt (list rows, when the text is known), recreate,
/// make video, share and download. Audio only shares and downloads (`Audio v1`).
fn item_actions<'a>(view: &FeedView<'_>, item: &FeedItem, prompt: Option<&String>) -> Vec<(Icon, &'a str, FeedAction)> {
let mut buttons = Vec::new();
if let Some(text) = prompt {
buttons.push((Icon::Copy, "Copy prompt", FeedAction::CopyPrompt(text.clone())));
}
if item.prompt_token.is_some() && item.kind != MediaKind::Audio {
buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone())));
}
if view.make_video && item.kind == MediaKind::Image {
buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone())));
}
buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone())));
buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone())));
buttons
}
/// What an item is called without its prompt: its title, else "Audio Generation" and the like.
fn fallback_title(item: &FeedItem) -> String {
item.title.clone().unwrap_or_else(|| format!("{} Generation", item.kind.label()))
}
fn kind_icon(kind: MediaKind) -> Icon {
match kind {
MediaKind::Image => Icon::Image,
MediaKind::Video => Icon::Video,
MediaKind::Audio => Icon::Music,
}
}
fn pending_row(ui: &mut Ui, rect: Rect, job: &PendingJob, now: i64, names: &dyn ModelNames) {
let thumb = Rect::from_min_size(rect.min + vec2(0.0, 12.0), vec2(LIST_THUMB, LIST_THUMB));
ui.painter().rect_filled(thumb, theme::RADIUS, Color32::from_white_alpha(8));
@@ -359,7 +427,7 @@ fn failed_row(ui: &mut Ui, id: Id, rect: Rect, job: &FailedJob, names: &dyn Mode
}
#[allow(clippy::too_many_arguments)]
fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, names: &dyn ModelNames, now: i64, actions: &mut Vec<FeedAction>) {
fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, audio: &mut AudioPlayer, names: &dyn ModelNames, now: i64, actions: &mut Vec<FeedAction>) {
let resp = ui.interact(rect, view.id.with(("row", &item.token)), Sense::click());
let hovered = resp.hovered();
if hovered {
@@ -367,7 +435,10 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
}
let thumb = Rect::from_min_size(rect.min + vec2(0.0, 12.0), vec2(LIST_THUMB, LIST_THUMB));
ui.painter().rect_filled(thumb, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4));
if let Some(Lookup::Ready(t)) = item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
let is_audio = item.kind == MediaKind::Audio;
if is_audio {
waveform::music_tile(ui, thumb, false);
} else if let Some(Lookup::Ready(t)) = item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
egui::Image::new(&t).uv(crate::prompt_box::deck::cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, thumb);
}
if view.selected.contains(&item.token) {
@@ -378,7 +449,8 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
let prompt = item.prompt_token.as_deref().and_then(|t| view.prompts.get(t));
match prompt {
Some(text) => {
let g = clamp_lines(ui, text, 14.0, theme::INK, w, 3);
// Audio keeps room for its player under the text.
let g = clamp_lines(ui, text, 14.0, theme::INK, w, if is_audio { 2 } else { 3 });
ui.painter().galley(pos2(x, thumb.top()), g, theme::INK);
},
None if item.prompt_token.is_some() => {
@@ -387,9 +459,13 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
}
},
None => {
ui.painter().text(pos2(x, thumb.top() + 8.0), Align2::LEFT_CENTER, format!("{} Generation", item.kind.label()), FontId::new(14.0, FontFamily::Proportional), theme::MUTED);
ui.painter().text(pos2(x, thumb.top() + 8.0), Align2::LEFT_CENTER, fallback_title(item), FontId::new(14.0, FontFamily::Proportional), theme::MUTED);
},
}
if is_audio {
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(Rect::from_min_size(pos2(x, thumb.top() + 44.0), vec2(w.min(LIST_PLAYER_MAX), 32.0))));
waveform::player(&mut child, view.id.with(("row-player", &item.token)), audio, &item.full_url, item.duration_secs, true);
}
let meta_y = thumb.bottom() - 10.0;
let mut mx = x;
if let Some(model) = item.model_id.as_deref() {
@@ -400,18 +476,7 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
ui.painter().text(pos2(rect.right(), thumb.top() + 8.0), Align2::RIGHT_CENTER, time_ago(item.created_at, now), FontId::new(11.0, FontFamily::Proportional), theme::FAINT);
let mut action_clicked = false;
if hovered && !view.selecting {
let mut buttons: Vec<(Icon, &str, FeedAction)> = Vec::new();
if let Some(text) = prompt {
buttons.push((Icon::Copy, "Copy prompt", FeedAction::CopyPrompt(text.clone())));
}
if item.prompt_token.is_some() {
buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone())));
}
if view.make_video && item.kind == MediaKind::Image {
buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone())));
}
buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone())));
buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone())));
let buttons = item_actions(view, item, prompt);
let bar = Rect::from_min_size(pos2(rect.right() - 28.0 * buttons.len() as f32, thumb.bottom() - 28.0), vec2(28.0 * buttons.len() as f32, 28.0));
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar).layout(egui::Layout::left_to_right(egui::Align::Center)));
child.spacing_mut().item_spacing.x = 0.0;
@@ -464,9 +529,15 @@ fn model_chip(ui: &Ui, left_center: egui::Pos2, model_id: &str, names: &dyn Mode
/// Darkens the bottom of `rect` (`bg-gradient-to-t from-black/70`), returning the faded band.
fn bottom_gradient(ui: &Ui, rect: Rect, height: f32, alpha: u8) -> Rect {
let band = Rect::from_min_max(pos2(rect.left(), (rect.bottom() - height).max(rect.top())), rect.max);
edge_fade(ui, rect, height, alpha, false)
}
/// Darkens `rect` toward its top or bottom edge over `height`, returning the faded band.
fn edge_fade(ui: &Ui, rect: Rect, height: f32, alpha: u8, top_edge: bool) -> Rect {
let band = if top_edge { Rect::from_min_max(rect.min, pos2(rect.right(), (rect.top() + height).min(rect.bottom()))) } else { Rect::from_min_max(pos2(rect.left(), (rect.bottom() - height).max(rect.top())), rect.max) };
let mut mesh = Mesh::default();
let (top, bottom) = (Color32::TRANSPARENT, Color32::from_black_alpha(alpha));
let (clear, dark) = (Color32::TRANSPARENT, Color32::from_black_alpha(alpha));
let (top, bottom) = if top_edge { (dark, clear) } else { (clear, dark) };
mesh.colored_vertex(band.left_top(), top);
mesh.colored_vertex(band.right_top(), top);
mesh.colored_vertex(band.right_bottom(), bottom);
+18 -6
View File
@@ -6,12 +6,14 @@ use std::collections::{HashMap, HashSet};
use crate::backend::wire::{Job, JobPhase, MediaFile, unix_secs};
use crate::backend::EnqueueMeta;
use crate::feed::types::{FailedJob, FeedItem, MediaKind, PendingJob};
use crate::models::Catalog;
use crate::models::{self, Catalog};
/// Failures older than this are history, not news.
const FAILURE_WINDOW_SECS: i64 = 24 * 60 * 60;
/// Thumbnails are requested at this width (`getMediaThumbnail(@512)`).
pub const THUMBNAIL_WIDTH: u32 = 512;
/// Animated previews are requested smaller: every frame becomes a texture.
const ANIMATED_WIDTH: u32 = 320;
/// Something the store wants the app to do or say after an update.
#[derive(Debug, PartialEq)]
@@ -54,7 +56,7 @@ impl FeedStore {
/// Shows the jobs a generate call just started, before the next poll catches up.
pub fn add_enqueued(&mut self, kind: MediaKind, job_tokens: &[String], meta: &EnqueueMeta, catalog: &Catalog) {
let now = chrono::Utc::now().timestamp();
let expected = catalog.find(&meta.model_id).map_or(30.0, |m| m.expected_secs);
let expected = catalog.find(&meta.model_id).map_or_else(|| models::expected_secs(&meta.model_id, kind), |m| m.expected_secs);
for token in job_tokens {
self.meta.insert(token.clone(), meta.clone());
self.seen_running.insert(token.clone());
@@ -79,7 +81,7 @@ impl FeedStore {
match job.status.phase() {
JobPhase::Running => {
self.seen_running.insert(job.job_token.clone());
let expected = model_id.as_deref().and_then(|id| catalog.find(id)).map_or(if kind == MediaKind::Video { 900.0 } else { 30.0 }, |m| m.expected_secs);
let expected = model_id.as_deref().and_then(|id| catalog.find(id)).map_or_else(|| models::expected_secs(model_id.as_deref().unwrap_or_default(), kind), |m| m.expected_secs);
pending.push(PendingJob { job_token: job.job_token.clone(), kind, prompt, model_id, created_at: unix_secs(&job.created_at), server_progress: Some(job.status.progress_percentage), expected_secs: expected, batch_count: meta.map_or(1, |m| m.batch_count) });
},
JobPhase::Failed => {
@@ -103,7 +105,8 @@ impl FeedStore {
};
// The finished card takes the pending card's place: date it by the job.
let created_at = unix_secs(&job.created_at);
self.push_item(FeedItem { token: result.entity_token.clone(), kind, thumbnail: result.media_links.thumbnail(THUMBNAIL_WIDTH), full_url: result.media_links.cdn_url.clone(), created_at, model_id: model_id.clone(), prompt_token: job.request.maybe_prompt_token.clone(), batch_token: result.maybe_batch_token.clone(), duration_secs: None });
let thumbnail = result.media_links.thumbnail(THUMBNAIL_WIDTH).filter(|_| kind != MediaKind::Audio);
self.push_item(FeedItem { token: result.entity_token.clone(), kind, title: None, thumbnail, full_url: result.media_links.cdn_url.clone(), created_at, model_id: model_id.clone(), prompt_token: job.request.maybe_prompt_token.clone(), batch_token: result.maybe_batch_token.clone(), duration_secs: None, animated: result.media_links.animated_preview(ANIMATED_WIDTH) });
if let Some(batch) = &result.maybe_batch_token {
notices.push(FeedNotice::LoadBatch { job_token: job.job_token.clone(), batch_token: batch.clone() });
}
@@ -207,19 +210,23 @@ pub fn feed_item(kind: MediaKind, file: &MediaFile) -> FeedItem {
kind: match file.media_class.as_deref() {
Some("video") => MediaKind::Video,
Some("image") => MediaKind::Image,
Some("audio") => MediaKind::Audio,
_ => kind,
},
thumbnail: file.media_links.thumbnail(THUMBNAIL_WIDTH),
title: file.maybe_title.clone().filter(|t| !t.is_empty()),
thumbnail: file.thumbnail(THUMBNAIL_WIDTH),
full_url: file.media_links.cdn_url.clone(),
created_at: unix_secs(&file.created_at),
model_id: file.maybe_model_type.clone().or_else(|| file.maybe_origin_model_type.clone()),
prompt_token: file.maybe_prompt_token.clone(),
batch_token: file.maybe_batch_token.clone(),
duration_secs: file.maybe_duration_millis.map(|ms| ms as f32 / 1000.0),
animated: file.media_links.animated_preview(ANIMATED_WIDTH),
}
}
/// Whether a job made an image or a video: by its model, else by its category.
/// Whether a job made an image, a video or audio: by its model, else by its category
/// (`useGenerationJobs` checks video before image before audio).
fn job_kind(job: &Job, catalog: &Catalog) -> Option<MediaKind> {
if let Some(model) = job.request.maybe_model_type.as_deref() {
if catalog.image.iter().any(|m| m.id == model) {
@@ -228,12 +235,17 @@ fn job_kind(job: &Job, catalog: &Catalog) -> Option<MediaKind> {
if catalog.video.iter().any(|m| m.id == model) {
return Some(MediaKind::Video);
}
if catalog.audio.iter().any(|m| m.id == model) {
return Some(MediaKind::Audio);
}
}
let category = job.request.inference_category.as_str();
if category.contains("video") {
Some(MediaKind::Video)
} else if category.contains("image") {
Some(MediaKind::Image)
} else if category.contains("audio") {
Some(MediaKind::Audio)
} else {
None
}
+9 -2
View File
@@ -4,6 +4,7 @@
pub enum MediaKind {
Image,
Video,
Audio,
}
impl MediaKind {
@@ -11,6 +12,7 @@ impl MediaKind {
match self {
MediaKind::Image => "Image",
MediaKind::Video => "Video",
MediaKind::Audio => "Audio",
}
}
@@ -18,17 +20,20 @@ impl MediaKind {
match self {
MediaKind::Image => "images",
MediaKind::Video => "videos",
MediaKind::Audio => "audio clips",
}
}
}
/// A finished image or video in the user's library.
/// A finished image, video or audio clip in the user's library.
#[derive(Clone, Debug)]
pub struct FeedItem {
/// The media file token.
pub token: String,
pub kind: MediaKind,
/// A thumbnail URL (512 px wide where the CDN can resize).
/// The file's title (`maybe_title`), when it has one.
pub title: Option<String>,
/// A thumbnail URL (512 px wide where the CDN can resize); audio has none.
pub thumbnail: Option<String>,
/// The original file.
pub full_url: String,
@@ -39,6 +44,8 @@ pub struct FeedItem {
pub prompt_token: Option<String>,
pub batch_token: Option<String>,
pub duration_secs: Option<f32>,
/// A video's animated preview (shown when previews play).
pub animated: Option<String>,
}
/// A generation the server is still working on.
+2
View File
@@ -21,6 +21,8 @@ fn main() -> eframe::Result {
if let Some(icon) = app_icon() {
viewport = viewport.with_icon(icon);
}
// The app draws its own title bar; macOS keeps its traffic lights over it.
viewport = if cfg!(target_os = "macos") { viewport.with_fullsize_content_view(true).with_titlebar_shown(false).with_title_shown(false) } else { viewport.with_decorations(false) };
let options = eframe::NativeOptions { viewport, persist_window: true, ..Default::default() };
eframe::run_native("ArtCraft", options, Box::new(|cc| Ok(Box::new(app::ArtcraftApp::new(cc)))))
}
+105 -7
View File
@@ -8,10 +8,15 @@ use crate::feed::types::MediaKind;
/// Default models (`defaultModelForPage.ts`).
pub const DEFAULT_IMAGE_MODEL: &str = "nano_banana_pro";
pub const DEFAULT_VIDEO_MODEL: &str = "seedance_2p0";
pub const DEFAULT_AUDIO_MODEL: &str = "suno_music";
/// The fallback prompt limit (`maxPromptLength ?? 1000`).
pub const DEFAULT_PROMPT_MAX: usize = 1000;
/// The model pickers group by family once they list this many models.
const GROUP_THRESHOLD: usize = 8;
/// Audio models that work from exactly one audio track (`AUDIO_MODELS_REQUIRING_AUDIO_REF`).
const AUDIO_MODELS_REQUIRING_AUDIO_REF: [&str; 2] = ["suno_remix", "suno_sample"];
/// The musical keys (`SoundsSettingsPopover`; there are no E keys): value, label, short label.
pub const MUSICAL_KEYS: [(&str, &str, &str); 13] = [("auto", "Auto", "Auto"), ("c_major", "C Major", "C"), ("c_minor", "C Minor", "Cm"), ("d_major", "D Major", "D"), ("d_minor", "D Minor", "Dm"), ("f_major", "F Major", "F"), ("f_minor", "F Minor", "Fm"), ("g_major", "G Major", "G"), ("g_minor", "G Minor", "Gm"), ("a_major", "A Major", "A"), ("a_minor", "A Minor", "Am"), ("b_major", "B Major", "B"), ("b_minor", "B Minor", "Bm")];
/// One generation model and what it accepts.
#[derive(Clone, Debug, Default)]
@@ -45,6 +50,8 @@ pub struct ModelInfo {
pub video_refs_max_secs: Option<u16>,
pub audio_refs_max: usize,
pub audio_refs_max_secs: Option<u16>,
/// `@Character` mentions (0 = not supported).
pub character_refs_max: usize,
pub duration_options: Vec<u16>,
pub duration_min: Option<u16>,
pub duration_max: Option<u16>,
@@ -54,10 +61,42 @@ pub struct ModelInfo {
pub bitrate_default: Option<String>,
pub output_formats: Vec<String>,
pub output_format_default: Option<String>,
/// Audio only.
pub audio: AudioCaps,
/// How long a generation usually takes (drives the estimated progress bar).
pub expected_secs: f32,
}
/// What an audio model takes besides the prompt and references.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct AudioCaps {
pub style_prompt: bool,
pub instrumental: bool,
pub keep_lyrics: bool,
pub loopable: bool,
pub bpm: bool,
pub musical_key: bool,
pub sample_rates: Vec<u32>,
pub sample_rate_default: Option<u32>,
pub speed: bool,
pub volume: bool,
pub pitch: bool,
/// Remixes and samples work from exactly one audio track.
pub requires_audio_ref: bool,
}
impl AudioCaps {
/// Whether the Tuning popover shows.
pub fn has_tuning(&self) -> bool {
!self.sample_rates.is_empty() || self.speed || self.volume || self.pitch
}
/// `picked` if the model offers it, else its default, else its first rate.
pub fn sample_rate(&self, picked: Option<u32>) -> Option<u32> {
picked.filter(|hz| self.sample_rates.contains(hz)).or(self.sample_rate_default).or_else(|| self.sample_rates.first().copied())
}
}
impl ModelInfo {
/// Batch sizes the count picker offers (`predefined` or `1..=max`).
pub fn batch_options(&self) -> Vec<u16> {
@@ -162,6 +201,7 @@ pub fn video_model(d: &artcraft_client::endpoints::omni_gen::models::video::omni
m.video_refs_max_secs = d.video_references_max_total_duration_seconds;
m.audio_refs_max = max(d.audio_references_supported, d.audio_references_max);
m.audio_refs_max_secs = d.audio_references_max_total_duration_seconds;
m.character_refs_max = max(d.character_references_supported, d.character_references_max);
m.aspect_ratios = strings(&d.aspect_ratio_options);
m.aspect_default = d.aspect_ratio_default.clone().map(String::from);
m.resolutions = strings(&d.resolution_options);
@@ -182,6 +222,22 @@ pub fn video_model(d: &artcraft_client::endpoints::omni_gen::models::video::omni
m
}
/// Builds a model from the server's audio listing.
pub fn audio_model(d: &crate::backend::wire::AudioModel) -> ModelInfo {
let mut m = base(&d.model, MediaKind::Audio, d.full_name.as_deref(), d.model_creator.clone());
if let Some(short) = d.extra_info_short.as_deref().filter(|s| !s.is_empty()) {
m.description = short.to_owned();
}
m.disabled = d.is_disabled;
m.prompt_supported = d.text_prompt_supported.unwrap_or(true);
m.prompt_max = None;
let max = |supported: bool, max: Option<u16>| if supported { max.map_or(1, usize::from) } else { 0 };
m.audio_refs_max = max(d.audio_references_supported, d.audio_references_max);
m.image_refs_max = max(d.image_references_supported, d.image_references_max);
m.audio = AudioCaps { style_prompt: d.style_prompt_supported, instrumental: d.instrumental_toggle_supported, keep_lyrics: d.keep_lyrics_supported, loopable: d.loopable_toggle_supported, bpm: d.bpm_supported, musical_key: d.musical_key_supported, sample_rates: d.sample_rate_hz_options.clone(), sample_rate_default: d.sample_rate_hz_default, speed: d.speed_supported, volume: d.volume_supported, pitch: d.pitch_supported, requires_audio_ref: AUDIO_MODELS_REQUIRING_AUDIO_REF.contains(&d.model.as_str()) };
m
}
fn strings<T: Clone + Into<String>>(options: &Option<Vec<T>>) -> Vec<String> {
options.as_deref().unwrap_or_default().iter().cloned().map(Into::into).collect()
}
@@ -209,7 +265,7 @@ fn base(id: &str, kind: MediaKind, full_name: Option<&str>, creator: Option<Stri
/// Families whose picker names the app curates (the static model lists' `selectorName`s).
fn has_curated_name(id: &str) -> bool {
const PREFIXES: [&str; 11] = ["flux_", "nano_banana", "gpt_image", "seedream", "seedance", "kling", "veo_", "sora", "happy_horse", "grok_imagine", "qwen"];
const PREFIXES: [&str; 13] = ["flux_", "nano_banana", "gpt_image", "seedream", "seedance", "kling", "veo_", "sora", "happy_horse", "grok_imagine", "qwen", "suno", "seed_audio"];
PREFIXES.iter().any(|p| id.starts_with(p))
}
@@ -289,12 +345,17 @@ fn description(id: &str) -> &'static str {
"grok_video" | "grok_imagine_video" => "Fastest video model",
"sora_2" => "Smart video model",
"veo_3" => "Slow, high-quality model",
"suno_music" => "Full songs from a text prompt",
"suno_remix" => "Remix an existing track",
"suno_sounds" => "Sound effects with beat control",
"suno_sample" => "Build a song from a sample",
"seed_audio_1p0" => "Sound generation with fine tuning",
_ => "",
}
}
/// Typical generation time (the static list's `progressBarTime`).
fn expected_secs(id: &str, kind: MediaKind) -> f32 {
/// Typical generation time (the static list's `progressBarTime`, else the feed's per-kind default).
pub fn expected_secs(id: &str, kind: MediaKind) -> f32 {
match id {
"grok_image" | "grok_imagine_image" | "flux_pro_1p1" | "flux_1_dev" | "flux_1_schnell" => 10.0,
"grok_video" | "grok_imagine_video" => 50.0,
@@ -304,19 +365,20 @@ fn expected_secs(id: &str, kind: MediaKind) -> f32 {
"gpt_image_1" | "gpt_image_1p5" | "gpt_image_2p5_flare" | "seedream_4" | "seedream_4p5" | "seedream_5_lite" => 60.0,
"gpt_image_2" | "gpt_image_2p5_sunburst" => 120.0,
_ if kind == MediaKind::Video => 900.0,
_ if kind == MediaKind::Audio => 120.0,
_ => 30.0,
}
}
/// The maker, by id prefix, when the server doesn't say.
pub fn creator_for(id: &str) -> &'static str {
const PREFIXES: [(&str, &str); 16] = [("flux", "black_forest_labs"), ("nano_banana", "google"), ("gemini", "google"), ("veo", "google"), ("imagen", "google"), ("gpt_image", "openai"), ("sora", "openai"), ("seedream", "bytedance"), ("seedance", "bytedance"), ("seededit", "bytedance"), ("kling", "kling"), ("grok", "grok"), ("midjourney", "midjourney"), ("qwen", "alibaba"), ("happy_horse", "alibaba"), ("minimax", "minimax")];
const PREFIXES: [(&str, &str); 18] = [("suno", "suno"), ("seed_audio", "bytedance"), ("flux", "black_forest_labs"), ("nano_banana", "google"), ("gemini", "google"), ("veo", "google"), ("imagen", "google"), ("gpt_image", "openai"), ("sora", "openai"), ("seedream", "bytedance"), ("seedance", "bytedance"), ("seededit", "bytedance"), ("kling", "kling"), ("grok", "grok"), ("midjourney", "midjourney"), ("qwen", "alibaba"), ("happy_horse", "alibaba"), ("minimax", "minimax")];
PREFIXES.iter().find(|(p, _)| id.starts_with(p)).map_or("artcraft", |(_, c)| c)
}
/// The picker family (`getModelFamilyName`).
fn family(id: &str) -> &'static str {
const FAMILIES: [(&str, &str); 22] = [("seedance", "Seedance"), ("kling", "Kling"), ("veo", "Veo"), ("sora", "Sora"), ("happy_horse", "Happy Horse"), ("wan", "Wan"), ("vidu", "Vidu"), ("preview", "Preview"), ("nano_banana", "Nano Banana"), ("gemini_25_flash", "Nano Banana"), ("seedream", "Seedream"), ("seededit", "SeedEdit"), ("gpt_image", "GPT Image"), ("flux", "Flux"), ("midjourney", "Midjourney"), ("grok", "Grok"), ("qwen", "Qwen"), ("recraft", "Recraft"), ("imagen", "Imagen"), ("minimax", "MiniMax"), ("marble", "World Labs"), ("suno", "Suno")];
const FAMILIES: [(&str, &str); 23] = [("seed_audio", "Seed Audio"), ("seedance", "Seedance"), ("kling", "Kling"), ("veo", "Veo"), ("sora", "Sora"), ("happy_horse", "Happy Horse"), ("wan", "Wan"), ("vidu", "Vidu"), ("preview", "Preview"), ("nano_banana", "Nano Banana"), ("gemini_25_flash", "Nano Banana"), ("seedream", "Seedream"), ("seededit", "SeedEdit"), ("gpt_image", "GPT Image"), ("flux", "Flux"), ("midjourney", "Midjourney"), ("grok", "Grok"), ("qwen", "Qwen"), ("recraft", "Recraft"), ("imagen", "Imagen"), ("minimax", "MiniMax"), ("marble", "World Labs"), ("suno", "Suno")];
FAMILIES.iter().find(|(p, _)| id.starts_with(p)).map_or("Other", |(_, f)| f)
}
@@ -386,13 +448,16 @@ fn video_page_models(models: &[ModelInfo]) -> Vec<ModelInfo> {
list
}
/// Both catalogs, for looking up names in the feed, and each create page's picker list.
/// Every catalog, for looking up names in the feed, and each create page's picker list.
#[derive(Default)]
pub struct Catalog {
pub image: Vec<ModelInfo>,
pub video: Vec<ModelInfo>,
pub audio: Vec<ModelInfo>,
pub image_page: Vec<ModelInfo>,
pub video_page: Vec<ModelInfo>,
/// The audio page's list: enabled, in the server's order.
pub audio_page: Vec<ModelInfo>,
}
impl Catalog {
@@ -406,8 +471,13 @@ impl Catalog {
self.video = models;
}
pub fn set_audio(&mut self, models: Vec<ModelInfo>) {
self.audio_page = models.iter().filter(|m| !m.disabled).cloned().collect();
self.audio = models;
}
pub fn find(&self, id: &str) -> Option<&ModelInfo> {
self.image.iter().chain(&self.video).find(|m| m.id == id)
self.image.iter().chain(&self.video).chain(&self.audio).find(|m| m.id == id)
}
}
@@ -502,6 +572,15 @@ pub fn quality_label(value: &str) -> String {
.to_owned()
}
/// Sample rate labels (`formatSampleRateHz`): 44100 → "44.1 kHz", 24000 → "24 kHz".
pub fn sample_rate_label(hz: u32) -> String {
if hz.is_multiple_of(1000) {
format!("{} kHz", hz / 1000)
} else {
format!("{:.1} kHz", hz as f32 / 1000.0)
}
}
/// Bitrate labels.
pub fn bitrate_label(value: &str) -> String {
match value {
@@ -530,6 +609,8 @@ mod tests {
assert_eq!(display_name("kling_3p0_pro"), "Kling 3.0 Pro");
assert_eq!(display_name("nano_banana"), "Nano Banana");
assert_eq!(display_name("seedance_2p0_bpu"), "Seedance 2.0 Plus Ultra");
assert_eq!(display_name("seed_audio_1p0"), "Seed Audio 1.0");
assert_eq!(display_name("suno_music"), "Suno Music");
}
#[test]
@@ -544,6 +625,8 @@ mod tests {
#[test]
fn falls_back_to_creator_by_prefix() {
assert_eq!(creator_for("seed_audio_1p0"), "bytedance");
assert_eq!(creator_for("suno_remix"), "suno");
assert_eq!(creator_for("seedream_4p5"), "bytedance");
assert_eq!(creator_for("veo_3"), "google");
assert_eq!(creator_for("something_new"), "artcraft");
@@ -573,6 +656,21 @@ mod tests {
assert_eq!(ModelInfo::resolve(None, &[], None), None);
}
#[test]
fn audio_listing_maps_capabilities() {
let d: crate::backend::wire::AudioModel = serde_json::from_value(serde_json::json!({ "model": "seed_audio_1p0", "model_creator": "artcraft", "audio_references_supported": true, "audio_references_max": 3, "image_references_supported": true, "sample_rate_hz_options": [8000, 24000, 44100], "sample_rate_hz_default": 24000, "speed_supported": true })).unwrap();
let m = audio_model(&d);
assert_eq!((m.name.as_str(), m.creator.as_str(), m.description.as_str()), ("Seed Audio 1.0", "bytedance", "Sound generation with fine tuning"));
assert_eq!((m.audio_refs_max, m.image_refs_max, m.prompt_max), (3, 1, None));
assert!(m.audio.has_tuning() && !m.audio.requires_audio_ref && !m.audio.style_prompt);
assert_eq!(m.audio.sample_rate(Some(44100)), Some(44100));
assert_eq!(m.audio.sample_rate(Some(11025)), Some(24000), "unsupported picks fall back to the default");
let remix = audio_model(&serde_json::from_value(serde_json::json!({ "model": "suno_remix", "audio_references_supported": true })).unwrap());
assert!(remix.audio.requires_audio_ref && !remix.audio.has_tuning());
assert_eq!((remix.audio_refs_max, remix.audio.sample_rate(None)), (1, None));
assert_eq!((sample_rate_label(44100).as_str(), sample_rate_label(24000).as_str()), ("44.1 kHz", "24 kHz"));
}
#[test]
fn batch_counts_snap_to_valid_values() {
let mut m = model("grok_imagine_image");
+402
View File
@@ -0,0 +1,402 @@
//! The characters dialog (`CharactersModal`): pick a saved character to mention it in the prompt,
//! make a new one from a reference image, rename or re-describe one, or delete it.
use std::path::PathBuf;
use std::sync::Arc;
use egui::{Align2, Color32, FontId, Id, Rect, Sense, Stroke, Ui, pos2, vec2};
use crate::backend::UploadedMedia;
use crate::backend::media_cache::{Lookup, MediaCache};
use crate::backend::wire::Character;
use crate::prompt_box::deck::cover_uv;
use crate::prompt_box::refs::{RefKind, RefMedia, RefStatus};
use crate::theme;
use crate::ui::icons::{self, Icon};
use crate::ui::widgets;
const COLUMNS: usize = 4;
const GAP: f32 = 12.0;
#[derive(Clone, PartialEq)]
enum View {
List,
Edit(String),
Create,
}
pub struct CharactersModal {
view: View,
confirm_delete: Option<(String, String)>,
name: String,
description: String,
image: Option<RefMedia>,
}
/// Something only the app can do for the dialog.
#[derive(Debug)]
pub enum CharactersAction {
Close,
/// Mention this character in the prompt.
Select {
token: String,
name: String,
},
Upload {
ref_id: u64,
path: PathBuf,
},
Create {
image_token: String,
name: String,
description: Option<String>,
},
Edit {
token: String,
name: String,
description: String,
},
Delete {
token: String,
name: String,
},
Preview(String),
Toast(String),
}
impl CharactersModal {
pub fn new() -> Self {
Self { view: View::List, confirm_delete: None, name: String::new(), description: String::new(), image: None }
}
/// The reference image being uploaded for a new character, if `ref_id` is it.
pub fn apply_upload(&mut self, ref_id: u64, result: Result<UploadedMedia, String>) -> Option<Result<(), String>> {
let image = self.image.as_mut().filter(|i| i.id == ref_id)?;
Some(match result {
Ok(media) => {
image.status = RefStatus::Ready;
image.token = Some(media.token);
image.full_url = media.full_url;
Ok(())
},
Err(err) => {
self.image = None;
Err(err)
},
})
}
pub fn show(&mut self, ctx: &egui::Context, characters: &[Character], pending: &[String], cache: &mut MediaCache) -> Vec<CharactersAction> {
let mut actions = Vec::new();
let width = (ctx.content_rect().width() - 80.0).min(760.0);
let close = widgets::modal(ctx, Id::new("characters"), width, |ui| match self.view.clone() {
View::List => self.list(ui, characters, pending, cache, &mut actions),
View::Edit(token) => match characters.iter().find(|c| c.token == token) {
Some(c) => self.edit(ui, c, cache, &mut actions),
None => self.view = View::List,
},
View::Create => self.create(ui, cache, &mut actions),
});
if close && self.confirm_delete.is_none() {
actions.push(CharactersAction::Close);
}
if let Some((token, name)) = self.confirm_delete.clone() {
let mut choice = None;
let dismissed = widgets::modal(ctx, Id::new("character-delete"), 420.0, |ui| {
ui.label(widgets::heading("Delete character?", 20.0));
ui.add_space(8.0);
ui.label(egui::RichText::new(format!("\u{201c}{name}\u{201d} will be removed from your characters.")).color(theme::MUTED));
ui.add_space(16.0);
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
if widgets::button(ui, None, "Delete", widgets::Kind::Danger, 0.0, theme::CONTROL_H).clicked() {
choice = Some(true);
}
if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() {
choice = Some(false);
}
});
});
if dismissed || choice.is_some() {
self.confirm_delete = None;
}
if choice == Some(true) {
actions.push(CharactersAction::Delete { token, name });
}
}
actions
}
fn header(&mut self, ui: &mut Ui, title: &str, back: bool, actions: &mut Vec<CharactersAction>) {
ui.horizontal(|ui| {
if back && widgets::ghost(ui, Icon::ChevronLeft, None, "Back", false).clicked() {
self.view = View::List;
}
ui.label(widgets::heading(title, 22.0));
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
if widgets::ghost(ui, Icon::X, None, "Close", false).clicked() {
actions.push(CharactersAction::Close);
}
});
});
ui.add_space(12.0);
}
fn list(&mut self, ui: &mut Ui, characters: &[Character], pending: &[String], cache: &mut MediaCache, actions: &mut Vec<CharactersAction>) {
self.header(ui, "Characters", false, actions);
let tile = (ui.available_width() - GAP * (COLUMNS - 1) as f32) / COLUMNS as f32;
let max_h = ui.ctx().content_rect().height() * 0.6;
egui::ScrollArea::vertical().max_height(max_h).auto_shrink([false, true]).show(ui, |ui| {
let total = 1 + pending.len() + characters.len();
let rows = total.div_ceil(COLUMNS);
let origin = ui.cursor().min;
ui.allocate_exact_size(vec2(ui.available_width(), rows as f32 * (tile + 30.0 + GAP)), Sense::hover());
let cell = |i: usize| Rect::from_min_size(origin + vec2((i % COLUMNS) as f32 * (tile + GAP), (i / COLUMNS) as f32 * (tile + 30.0 + GAP)), vec2(tile, tile));
// "Create New" first.
let create = cell(0);
let resp = ui.interact(create, Id::new("character-create"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, "Create New"));
ui.painter().rect_filled(create, theme::RADIUS, theme::fade(theme::CONTROLS, if resp.hovered() { 0.7 } else { 0.4 }));
icons::dashed_rect(ui.painter(), create.shrink(1.0), Stroke::new(1.5, theme::LINE_DASHED), 5.0, 4.0);
icons::paint(ui.painter(), Rect::from_center_size(create.center() - vec2(0.0, 10.0), vec2(22.0, 22.0)), Icon::Plus, theme::INK);
ui.painter().text(create.center() + vec2(0.0, 16.0), Align2::CENTER_CENTER, "Create New", FontId::new(13.0, theme::medium()), theme::INK);
if resp.clicked() {
self.name.clear();
self.description.clear();
self.image = None;
self.view = View::Create;
}
for (i, name) in pending.iter().enumerate() {
let r = cell(1 + i);
ui.painter().rect_filled(r, theme::RADIUS, theme::WASH);
icons::paint_spinner(ui, Rect::from_center_size(r.center(), vec2(24.0, 24.0)), theme::MUTED);
caption(ui, r, name, "Creating...");
}
for (i, c) in characters.iter().enumerate() {
let r = cell(1 + pending.len() + i);
let resp = ui.interact(r, Id::new(("character", &c.token)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, &c.name));
thumbnail(ui, r, c.avatar_url().as_deref(), cache);
if resp.hovered() {
ui.painter().rect_stroke(r, theme::RADIUS, Stroke::new(2.0, theme::fade(theme::ACCENT_400, 0.7)), egui::StrokeKind::Inside);
}
caption(ui, r, &c.name, "");
let mut clicked_tool = false;
if ui.rect_contains_pointer(r) {
let tools = Rect::from_min_size(pos2(r.right() - 64.0, r.top() + 6.0), vec2(58.0, 28.0));
ui.painter().rect_filled(tools, theme::RADIUS, Color32::from_black_alpha(160));
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(tools.shrink(1.0)).layout(egui::Layout::left_to_right(egui::Align::Center)));
child.spacing_mut().item_spacing.x = 0.0;
if widgets::overlay_icon(&mut child, Icon::Pencil, "Edit character").clicked() {
self.name = c.name.clone();
self.description = c.maybe_description.clone().unwrap_or_default();
self.view = View::Edit(c.token.clone());
clicked_tool = true;
}
if widgets::overlay_icon(&mut child, Icon::Trash, "Delete character").clicked() {
self.confirm_delete = Some((c.token.clone(), c.name.clone()));
clicked_tool = true;
}
}
if resp.clicked() && !clicked_tool {
actions.push(CharactersAction::Select { token: c.token.clone(), name: c.name.clone() });
}
}
});
}
fn edit(&mut self, ui: &mut Ui, c: &Character, cache: &mut MediaCache, actions: &mut Vec<CharactersAction>) {
self.header(ui, "Edit Character", true, actions);
ui.horizontal_top(|ui| {
ui.vertical(|ui| {
let (r, _) = ui.allocate_exact_size(vec2(240.0, 240.0), Sense::hover());
thumbnail(ui, r, c.full_url().or_else(|| c.avatar_url()).as_deref(), cache);
if let Some(url) = c.full_url() {
if ui.add(egui::Button::new(egui::RichText::new("View full size").size(12.0).color(theme::ACCENT_INK)).frame(false)).clicked() {
actions.push(CharactersAction::Preview(url));
}
}
});
ui.add_space(16.0);
ui.vertical(|ui| {
fields(ui, &mut self.name, &mut self.description);
ui.add_space(16.0);
ui.horizontal(|ui| {
if widgets::button(ui, None, "Save", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() {
if self.name.trim().is_empty() {
actions.push(CharactersAction::Toast("Name cannot be empty".to_owned()));
} else {
actions.push(CharactersAction::Edit { token: c.token.clone(), name: self.name.trim().to_owned(), description: self.description.trim().to_owned() });
self.view = View::List;
}
}
if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() {
self.view = View::List;
}
});
});
});
}
fn create(&mut self, ui: &mut Ui, cache: &mut MediaCache, actions: &mut Vec<CharactersAction>) {
self.header(ui, "New Character", true, actions);
ui.horizontal_top(|ui| {
ui.vertical(|ui| {
widgets::hud(ui, "Reference", theme::FAINT);
ui.add_space(6.0);
let (r, resp) = ui.allocate_exact_size(vec2(240.0, 240.0), Sense::click());
match &self.image {
Some(img) => {
thumbnail(ui, r, img.preview.as_deref(), cache);
if img.status == RefStatus::Uploading {
ui.painter().rect_filled(r, theme::RADIUS, Color32::from_black_alpha(110));
icons::paint_spinner(ui, Rect::from_center_size(r.center(), vec2(28.0, 28.0)), Color32::WHITE);
}
},
None => {
ui.painter().rect_filled(r, theme::RADIUS, theme::fade(theme::CONTROLS, if resp.hovered() { 0.7 } else { 0.4 }));
icons::dashed_rect(ui.painter(), r.shrink(1.0), Stroke::new(1.5, theme::LINE_DASHED), 5.0, 4.0);
icons::paint(ui.painter(), Rect::from_center_size(r.center() - vec2(0.0, 12.0), vec2(24.0, 24.0)), Icon::Upload, theme::INK);
ui.painter().text(r.center() + vec2(0.0, 18.0), Align2::CENTER_CENTER, "Upload reference image", FontId::new(13.0, theme::medium()), theme::INK);
},
}
if resp.on_hover_cursor(egui::CursorIcon::PointingHand).clicked() {
if let Some(path) = crate::pages::common::pick_files(&[RefKind::Image], false).into_iter().next() {
match std::fs::read(&path) {
Ok(bytes) => {
let mut img = RefMedia::uploading(RefKind::Image, None);
let key = format!("local://{}", img.id);
cache.insert_bytes(ui.ctx(), &key, Arc::from(bytes));
img.preview = Some(key);
actions.push(CharactersAction::Upload { ref_id: img.id, path });
self.image = Some(img);
},
Err(err) => actions.push(CharactersAction::Toast(format!("Failed to upload {}: {err}", crate::pages::common::file_name(&path)))),
}
}
}
});
ui.add_space(16.0);
ui.vertical(|ui| {
fields(ui, &mut self.name, &mut self.description);
ui.add_space(16.0);
let uploading = self.image.as_ref().is_some_and(|i| i.status == RefStatus::Uploading);
ui.horizontal(|ui| {
if ui.add_enabled_ui(!uploading, |ui| widgets::button(ui, Some(Icon::Sparkles), "Create", widgets::Kind::Primary, 0.0, theme::CONTROL_H)).inner.clicked() {
match (self.name.trim(), self.image.as_ref().and_then(|i| i.token.clone())) {
("", _) => actions.push(CharactersAction::Toast("Please enter a character name".to_owned())),
(_, None) => actions.push(CharactersAction::Toast("Please upload a reference image".to_owned())),
(name, Some(image_token)) => {
let description = Some(self.description.trim().to_owned()).filter(|d| !d.is_empty());
actions.push(CharactersAction::Create { image_token, name: name.to_owned(), description });
self.view = View::List;
},
}
}
if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() {
self.view = View::List;
}
});
});
});
}
}
/// The name and description inputs shared by create and edit.
fn fields(ui: &mut Ui, name: &mut String, description: &mut String) {
widgets::hud(ui, "Character name", theme::FAINT);
ui.add_space(4.0);
input_frame(ui, |ui| ui.add(egui::TextEdit::singleline(name).frame(egui::Frame::NONE).desired_width(f32::INFINITY).hint_text(egui::RichText::new("Character name").color(theme::FAINT))));
ui.add_space(10.0);
widgets::hud(ui, "Description", theme::FAINT);
ui.add_space(4.0);
input_frame(ui, |ui| ui.add(egui::TextEdit::multiline(description).frame(egui::Frame::NONE).desired_width(f32::INFINITY).desired_rows(4).hint_text(egui::RichText::new("Description...").color(theme::FAINT))));
}
fn input_frame(ui: &mut Ui, add: impl FnOnce(&mut Ui) -> egui::Response) {
egui::Frame::new().fill(theme::SUNKEN).stroke(theme::hairline()).corner_radius(theme::RADIUS).inner_margin(egui::Margin::symmetric(10, 8)).show(ui, |ui| add(ui));
}
fn thumbnail(ui: &Ui, rect: Rect, url: Option<&str>, cache: &mut MediaCache) {
ui.painter().rect_filled(rect, theme::RADIUS, theme::WASH);
match url.map(|u| cache.get(ui.ctx(), u)) {
Some(Lookup::Ready(t)) => {
egui::Image::new(&t).uv(cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, rect);
},
Some(Lookup::Loading) => icons::paint_spinner(ui, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), theme::FAINT),
_ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), Icon::User, theme::FAINT),
}
}
/// The name under a tile (and a muted note after it).
fn caption(ui: &Ui, tile: Rect, name: &str, note: &str) {
let y = tile.bottom() + 15.0;
let galley = ui.painter().layout_no_wrap(name.to_owned(), FontId::new(13.0, theme::medium()), theme::INK);
let w = galley.size().x.min(tile.width());
ui.painter().with_clip_rect(Rect::from_min_size(pos2(tile.left(), y - 10.0), vec2(tile.width(), 20.0))).galley(pos2(tile.left(), y - galley.size().y / 2.0), galley, theme::INK);
if !note.is_empty() {
ui.painter().text(pos2(tile.left() + w + 6.0, y), Align2::LEFT_CENTER, note, FontId::new(11.5, egui::FontFamily::Proportional), theme::MUTED);
}
}
#[cfg(test)]
mod tests {
use super::*;
use egui_kittest::Harness;
use egui_kittest::kittest::Queryable;
struct State {
_rt: tokio::runtime::Runtime,
cache: MediaCache,
modal: CharactersModal,
characters: Vec<Character>,
actions: Vec<String>,
fonts_ready: bool,
}
fn harness() -> Harness<'static, State> {
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().expect("runtime");
let cache = MediaCache::new(rt.handle().clone(), reqwest::Client::new());
let characters = vec![Character { token: "c1".into(), name: "Mira".into(), ..Default::default() }];
let state = State { _rt: rt, cache, modal: CharactersModal::new(), characters, actions: Vec::new(), fonts_ready: false };
let mut harness = Harness::builder().with_size(egui::vec2(1000.0, 800.0)).build_ui_state(
|ui, s: &mut State| {
// The harness draws one frame while building: fonts are installed then, used from the next.
if !s.fonts_ready {
crate::theme::apply(ui.ctx());
s.fonts_ready = true;
return;
}
let actions = s.modal.show(ui.ctx(), &s.characters, &[], &mut s.cache);
s.actions.extend(actions.iter().map(|a| format!("{a:?}")));
},
state,
);
harness.run();
harness
}
#[test]
fn picking_a_character_mentions_it() {
let mut harness = harness();
harness.get_by_label("Mira").click();
harness.run();
assert!(harness.state().actions.iter().any(|a| a.contains("Select") && a.contains("Mira")), "{:?}", harness.state().actions);
}
#[test]
fn creating_needs_a_name_and_a_reference() {
let mut harness = harness();
harness.get_by_label("Create New").click();
harness.run();
harness.get_by_label("Create").click();
harness.run();
assert_eq!(harness.state().actions, [r#"Toast("Please enter a character name")"#]);
harness.get_by_role(egui::accesskit::Role::TextInput).focus();
harness.run();
harness.get_by_role(egui::accesskit::Role::TextInput).type_text("Mira");
harness.run();
harness.get_by_label("Create").click();
harness.run();
assert_eq!(harness.state().actions.last().map(String::as_str), Some(r#"Toast("Please upload a reference image")"#));
}
}
@@ -153,7 +153,7 @@ impl LibraryPicker {
}
fn picks(&self) -> Vec<RefMedia> {
self.selected.iter().filter_map(|t| self.items.iter().find(|i| i.token == *t)).map(|f| RefMedia::from_library(self.kind, f.token.clone(), f.media_links.thumbnail(256), Some(f.media_links.cdn_url.clone()), f.maybe_duration_millis.map_or(0.0, |ms| ms as f32 / 1000.0))).collect()
self.selected.iter().filter_map(|t| self.items.iter().find(|i| i.token == *t)).map(|f| RefMedia::from_library(self.kind, f.token.clone(), f.thumbnail(256), Some(f.media_links.cdn_url.clone()), f.maybe_duration_millis.map_or(0.0, |ms| ms as f32 / 1000.0))).collect()
}
}
@@ -162,7 +162,7 @@ fn tile_ui(ui: &mut egui::Ui, rect: Rect, item: &MediaFile, kind: RefKind, attac
let resp = ui.interact(rect, Id::new(("picker-tile", &item.token)), if attached { Sense::hover() } else { Sense::click() });
let p = ui.painter();
p.rect_filled(rect, 0.0, theme::fade(theme::CONTROLS, 0.6));
match item.media_links.thumbnail(256).map(|u| cache.get(ui.ctx(), &u)) {
match item.thumbnail(256).map(|u| cache.get(ui.ctx(), &u)) {
Some(Lookup::Ready(t)) => {
let image = egui::Image::new(&t).uv(cover_uv(t.size_vec2()));
let image = if attached { image.tint(Color32::from_gray(90)) } else { image };
+55 -8
View File
@@ -1,23 +1,30 @@
//! The lightbox (`lightbox-modal`): the media large on black, prev/next through the feed, and a
//! side column with the prompt, details and actions.
//! The lightbox (`lightbox-modal`): the media large on black (audio as a player panel), prev/next
//! through the feed, and a side column with the prompt, details and actions.
use egui::{Align2, Color32, FontId, Id, Key, Rect, Sense, Ui, pos2, vec2};
use crate::backend::audio::AudioPlayer;
use crate::backend::media_cache::{Lookup, MediaCache};
use crate::backend::video::{self, VideoPlayer};
use crate::backend::wire::Prompt;
use crate::feed::grid::ModelNames;
use crate::feed::types::{FeedItem, MediaKind};
use crate::theme;
use crate::ui::icons::{self, Icon};
use crate::ui::{creator_icons, widgets};
use crate::ui::{creator_icons, waveform, widgets};
const SIDE_WIDTH: f32 = 280.0;
/// The audio panel's widest (`max-w-xl`) and its padding.
const AUDIO_PANEL_MAX: f32 = 576.0;
const AUDIO_PANEL_PAD: f32 = 24.0;
pub struct Lightbox {
pub kind: MediaKind,
pub token: String,
confirm_delete: bool,
prompt_expanded: bool,
/// The in-app player for videos (when FFmpeg is installed).
player: Option<VideoPlayer>,
}
#[derive(Debug, PartialEq)]
@@ -35,10 +42,11 @@ pub enum LightboxAction {
impl Lightbox {
pub fn new(kind: MediaKind, token: String) -> Self {
Self { kind, token, confirm_delete: false, prompt_expanded: false }
Self { kind, token, confirm_delete: false, prompt_expanded: false, player: None }
}
pub fn show(&mut self, ctx: &egui::Context, item: &FeedItem, order: &[String], prompt: Option<&Prompt>, names: &dyn ModelNames, cache: &mut MediaCache) -> Vec<LightboxAction> {
#[allow(clippy::too_many_arguments)]
pub fn show(&mut self, ctx: &egui::Context, item: &FeedItem, order: &[String], prompt: Option<&Prompt>, names: &dyn ModelNames, cache: &mut MediaCache, audio: &mut AudioPlayer) -> Vec<LightboxAction> {
let mut actions = Vec::new();
let screen = ctx.content_rect();
let size = vec2((screen.width() - 80.0).min(1200.0), (screen.height() - 80.0).min(760.0));
@@ -62,7 +70,7 @@ impl Lightbox {
ui.horizontal_top(|ui| {
let media_rect = Rect::from_min_size(ui.cursor().min, vec2(ui.available_width() - SIDE_WIDTH - 16.0, size.y - 32.0));
ui.allocate_rect(media_rect, Sense::hover());
self.media(ui, media_rect, item, cache, &prev, &next, &mut actions);
self.media(ui, media_rect, item, cache, audio, &prev, &next, &mut actions);
ui.add_space(16.0);
ui.vertical(|ui| {
ui.set_width(SIDE_WIDTH);
@@ -80,7 +88,29 @@ impl Lightbox {
}
#[allow(clippy::too_many_arguments)]
fn media(&self, ui: &mut Ui, rect: Rect, item: &FeedItem, cache: &mut MediaCache, prev: &Option<String>, next: &Option<String>, actions: &mut Vec<LightboxAction>) {
fn media(&mut self, ui: &mut Ui, rect: Rect, item: &FeedItem, cache: &mut MediaCache, audio: &mut AudioPlayer, prev: &Option<String>, next: &Option<String>, actions: &mut Vec<LightboxAction>) {
if item.kind == MediaKind::Audio {
self.player = None;
ui.painter().rect_filled(rect, 0.0, Color32::BLACK);
audio_panel(ui, rect, item, audio);
self.nav_buttons(ui, rect, prev, next, actions);
return;
}
if item.kind == MediaKind::Video && video::ffmpeg_available() {
if self.player.as_ref().is_none_or(|p| p.url() != item.full_url) {
self.player = Some(VideoPlayer::new(&item.full_url));
}
let poster = match item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
Some(Lookup::Ready(t)) => Some(t),
_ => None,
};
if let Some(player) = &mut self.player {
player.ui(ui, rect, poster.as_ref());
}
self.nav_buttons(ui, rect, prev, next, actions);
return;
}
self.player = None;
ui.painter().rect_filled(rect, 0.0, Color32::BLACK);
let url = if item.kind == MediaKind::Image { Some(item.full_url.as_str()) } else { item.thumbnail.as_deref() };
match url.map(|u| cache.get(ui.ctx(), u)) {
@@ -109,6 +139,11 @@ impl Lightbox {
actions.push(LightboxAction::Play(item.full_url.clone()));
}
}
self.nav_buttons(ui, rect, prev, next, actions);
}
/// Previous / next arrows at the media's sides (shown while hovering it).
fn nav_buttons(&self, ui: &mut Ui, rect: Rect, prev: &Option<String>, next: &Option<String>, actions: &mut Vec<LightboxAction>) {
let hovering = ui.rect_contains_pointer(rect);
for (target, left, tip) in [(prev, true, "Previous item"), (next, false, "Next item")] {
let Some(token) = target else {
@@ -214,7 +249,8 @@ impl Lightbox {
ui.add_space(16.0);
let w = (ui.available_width() - 8.0) / 2.0;
if item.prompt_token.is_some() && widgets::button(ui, Some(Icon::RotateCw), "Recreate", widgets::Kind::Primary, ui.available_width(), theme::CONTROL_H).clicked() {
// Audio can't be recreated yet (`Audio v1`).
if item.prompt_token.is_some() && item.kind != MediaKind::Audio && widgets::button(ui, Some(Icon::RotateCw), "Recreate", widgets::Kind::Primary, ui.available_width(), theme::CONTROL_H).clicked() {
actions.push(LightboxAction::Recreate);
}
ui.horizontal_wrapped(|ui| {
@@ -259,6 +295,17 @@ impl Lightbox {
}
}
/// Audio in the lightbox: a bordered panel with the music tile and the full-size player.
fn audio_panel(ui: &mut Ui, rect: Rect, item: &FeedItem, audio: &mut AudioPlayer) {
let width = (rect.width() - 48.0).min(AUDIO_PANEL_MAX);
let panel = Rect::from_center_size(rect.center(), vec2(width, AUDIO_PANEL_PAD * 2.0 + 64.0 + 20.0 + 44.0));
ui.painter().rect(panel, theme::RADIUS, theme::fade(theme::CONTROLS, 0.6), theme::hairline(), egui::StrokeKind::Inside);
waveform::music_tile(ui, Rect::from_center_size(pos2(panel.center().x, panel.top() + AUDIO_PANEL_PAD + 32.0), vec2(64.0, 64.0)), false);
let bar = Rect::from_min_max(pos2(panel.left() + AUDIO_PANEL_PAD, panel.bottom() - AUDIO_PANEL_PAD - 44.0), pos2(panel.right() - AUDIO_PANEL_PAD, panel.bottom() - AUDIO_PANEL_PAD));
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar));
waveform::player(&mut child, Id::new(("lightbox-player", &item.token)), audio, &item.full_url, item.duration_secs, false);
}
fn info_row(ui: &mut Ui, label: &str, value: impl FnOnce(&mut Ui)) {
ui.horizontal(|ui| {
let (rect, _) = ui.allocate_exact_size(vec2(96.0, 20.0), Sense::hover());
+1
View File
@@ -1,3 +1,4 @@
pub mod account;
pub mod characters;
pub mod library_picker;
pub mod lightbox;
+61 -7
View File
@@ -7,6 +7,7 @@ use std::sync::Arc;
use egui::{Align2, Color32, Id, Mesh, Order, Rect, Ui, pos2, vec2};
use serde_json::{Map, Value};
use crate::backend::audio::AudioPlayer;
use crate::backend::media_cache::MediaCache;
use crate::backend::{Backend, Modality, UploadedMedia};
use crate::feed::grid::{self, FeedAction, FeedView, RatioCache, ViewMode};
@@ -29,28 +30,55 @@ pub struct Env<'a> {
pub ctx: egui::Context,
pub backend: &'a Backend,
pub cache: &'a mut MediaCache,
/// Plays reference clips and feed tracks.
pub audio: &'a mut AudioPlayer,
pub toasts: &'a mut Toasts,
pub catalog: &'a Catalog,
pub signed_in: bool,
pub enter_to_generate: bool,
pub view_mode: ViewMode,
pub autoplay: bool,
pub ratios: &'a mut RatioCache,
pub prompts: &'a std::collections::HashMap<String, String>,
pub requests: &'a mut Vec<AppRequest>,
/// The user's characters, newest first.
pub characters: &'a [crate::backend::wire::Character],
}
/// Things only the app can do.
#[derive(Debug)]
pub enum AppRequest {
SignIn,
Open { kind: MediaKind, token: String },
PickFromLibrary { kind: RefKind, slot: ImageSlot, max: usize, page: MediaKind },
MakeVideo { token: String },
Recreate { kind: MediaKind, token: String },
Open {
kind: MediaKind,
token: String,
},
PickFromLibrary {
kind: RefKind,
slot: ImageSlot,
max: usize,
page: MediaKind,
},
MakeVideo {
token: String,
},
Recreate {
kind: MediaKind,
token: String,
},
Share(String),
Download(Vec<String>),
CopyText(String),
Preview(String),
/// Play or stop an audio reference.
ToggleAudio {
ref_id: u64,
url: String,
},
/// Open a file in the system's player.
PlayExternally(String),
/// Open the characters dialog (it mentions the picked one in the video prompt).
OpenCharacters,
LoadMoreLibrary(FeedKey),
}
@@ -59,6 +87,7 @@ pub enum AppRequest {
pub enum FeedKey {
Image,
Video,
Audio,
Library,
}
@@ -67,6 +96,7 @@ impl FeedKey {
match kind {
MediaKind::Image => FeedKey::Image,
MediaKind::Video => FeedKey::Video,
MediaKind::Audio => FeedKey::Audio,
}
}
@@ -75,9 +105,33 @@ impl FeedKey {
match self {
FeedKey::Image => "image",
FeedKey::Video => "video",
FeedKey::Library => "image,video",
FeedKey::Audio => "audio",
FeedKey::Library => "image,video,audio",
}
}
/// What the feed's items default to (the library mixes kinds; each file says its own).
pub fn kind(self) -> MediaKind {
match self {
FeedKey::Image | FeedKey::Library => MediaKind::Image,
FeedKey::Video => MediaKind::Video,
FeedKey::Audio => MediaKind::Audio,
}
}
/// The backend's name for this feed's library requests.
pub fn name(self) -> &'static str {
match self {
FeedKey::Image => "image",
FeedKey::Video => "video",
FeedKey::Audio => "audio",
FeedKey::Library => "library",
}
}
pub fn from_name(name: &str) -> Option<Self> {
[FeedKey::Image, FeedKey::Video, FeedKey::Audio, FeedKey::Library].into_iter().find(|k| k.name() == name)
}
}
/// The page around the feed: the splash empty state (or the feed), a fade behind the prompt box,
@@ -87,9 +141,9 @@ pub fn create_shell(ui: &mut Ui, env: &mut Env<'_>, kind: MediaKind, title: &str
let area = ui.max_rect();
let bottom_offset = *box_height + 40.0;
if feed.has_content() {
let view = FeedView { id: Id::new(("feed", kind.label())), mode: env.view_mode, pending: &feed.pending, failed: &feed.failed, items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: kind == MediaKind::Image, bottom_padding: bottom_offset + 24.0 };
let view = FeedView { id: Id::new(("feed", kind.label())), mode: env.view_mode, pending: &feed.pending, failed: &feed.failed, items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: kind == MediaKind::Image, autoplay: env.autoplay, bottom_padding: bottom_offset + 24.0 };
let mut feed_ui = ui.new_child(egui::UiBuilder::new().max_rect(area.shrink2(vec2(12.0, 0.0))));
let actions = grid::show(&mut feed_ui, &view, env.cache, env.ratios, env.catalog);
let actions = grid::show(&mut feed_ui, &view, env.cache, env.ratios, env.audio, env.catalog);
handle_feed_actions(env, kind, feed, actions);
bottom_fade(ui, area);
} else {
+882
View File
@@ -0,0 +1,882 @@
//! Create Audio (`CreateAudio` + `PromptBoxAudio`): a song, sound or sample from a prompt, with
//! the model's options (style, instrumental, lyrics, loop, beat & key, tuning), audio or image
//! references, the cost, and the audio feed.
use std::ops::RangeInclusive;
use std::path::PathBuf;
use egui::{Align2, Color32, FontId, Id, Rect, Response, Sense, Stroke, TextureHandle, Ui, pos2, vec2};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use crate::backend::media_cache::Lookup;
use crate::backend::{EnqueueMeta, Modality};
use crate::feed::store::FeedStore;
use crate::feed::types::MediaKind;
use crate::models::{self, AudioCaps, Catalog, DEFAULT_AUDIO_MODEL, MUSICAL_KEYS, ModelInfo};
use crate::pages::common::{self, AppRequest, CostState, Env};
use crate::prompt_box::deck::cover_uv;
use crate::prompt_box::model_selector::model_selector;
use crate::prompt_box::pickers;
use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, RefStatus, References};
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot};
use crate::theme;
use crate::ui::icons::{self, Icon};
use crate::ui::toast::Toasts;
use crate::ui::widgets::{self, Leading};
const PLACEHOLDER: &str = "Describe the music or sound you want...";
const STYLE_PLACEHOLDER: &str = "Style: e.g. dreamy synth-pop, female vocals";
/// The audio references' total length cap (`AUDIO_REF_MAX_DURATION_SECONDS`).
const MAX_REF_SECS: f32 = 600.0;
/// Beat & Key and Tuning (`SoundsSettingsPopover`, `AudioTuningPopover`).
const BPM_RANGE: RangeInclusive<u16> = 40..=240;
const DEFAULT_BPM: u16 = 120;
const SPEED_RANGE: RangeInclusive<f32> = 0.5..=2.0;
const VOLUME_RANGE: RangeInclusive<f32> = 0.5..=2.0;
const PITCH_RANGE: RangeInclusive<i8> = -12..=12;
const POPOVER_WIDTH: f32 = 240.0;
/// Shown when generate fails on the server's side (`OMNI_GENERATE_OUTAGE_MESSAGE`).
const OUTAGE_MESSAGE: &str = "Couldn't start generation - the service is temporarily unavailable. Try again shortly.";
/// What the page remembers between launches.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(default)]
pub struct AudioSettings {
pub prompt: String,
pub style_prompt: String,
pub model: Option<String>,
pub instrumental: bool,
pub keep_lyrics: bool,
pub loopable: bool,
/// `None` = Auto.
pub bpm: Option<u16>,
pub musical_key: String,
/// `None` = the model's default.
pub sample_rate_hz: Option<u32>,
pub speed: f32,
pub volume: f32,
/// Semitones.
pub pitch: i8,
}
impl Default for AudioSettings {
fn default() -> Self {
Self { prompt: String::new(), style_prompt: String::new(), model: None, instrumental: false, keep_lyrics: false, loopable: false, bpm: None, musical_key: "auto".to_owned(), sample_rate_hz: None, speed: 1.0, volume: 1.0, pitch: 0 }
}
}
#[derive(Default)]
pub struct CreateAudioPage {
pub settings: AudioSettings,
pub refs: References,
pub feed: FeedStore,
pub cost: CostState,
pub generating: bool,
box_state: PromptBoxState,
box_height: f32,
}
/// What the reference row asks for.
#[derive(Debug, PartialEq)]
enum RowAction {
AddAudio,
AudioFromLibrary,
AddImage,
Remove(u64),
Toggle(u64),
}
impl CreateAudioPage {
pub fn new(settings: AudioSettings) -> Self {
let mut page = Self { settings, box_height: 138.0, ..Default::default() };
page.box_state.editor.focus_requested = true;
page
}
pub fn ui(&mut self, ui: &mut Ui, env: &mut Env<'_>) {
let model = common::selected_model(&env.catalog.audio_page, self.settings.model.as_deref(), DEFAULT_AUDIO_MODEL).cloned();
if let Some(m) = &model {
let fields = self.fields(m, true);
self.cost.update(ui, env.backend, Modality::Audio, &fields);
}
let mut feed = std::mem::take(&mut self.feed);
let mut box_height = self.box_height;
common::create_shell(ui, env, MediaKind::Audio, "Create Audio", "Describe a song, a sound, or a sample", &mut feed, &mut box_height, |ui, env| self.prompt_box(ui, env, model.as_ref()));
self.feed = feed;
self.box_height = box_height;
}
fn prompt_box(&mut self, ui: &mut Ui, env: &mut Env<'_>, model: Option<&ModelInfo>) {
let missing_ref = model.is_some_and(|m| self.missing_required_ref(m));
let accepts: Vec<RefKind> = model.map_or_else(Vec::new, |m| [(RefKind::Audio, m.audio_refs_max), (RefKind::Image, m.image_refs_max)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect());
let props = PromptBoxProps { id: Id::new("audio-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: None, deck: DeckMode::None, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !missing_ref, generating: self.generating, generate_tooltip: "Generate", warning: missing_ref.then_some("Audio track required"), banner: None, extra_input: !self.settings.style_prompt.is_empty() };
let playing = self.refs.audios.iter().map(|r| r.id).find(|id| env.audio.is_playing(&ref_key(*id)));
let image = self.refs.images.first().and_then(|r| r.preview.as_deref()).and_then(|key| match env.cache.get(&env.ctx, key) {
Lookup::Ready(t) => Some(t),
_ => None,
});
let models = &env.catalog.audio_page;
let mut prompt = std::mem::take(&mut self.settings.prompt);
let settings = &mut self.settings;
let refs = &self.refs;
let mut picked_model = None;
let mut row_actions = Vec::new();
let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) {
(Slot::ToolbarLeft, _) => {
if let Some(id) = model_selector(ui, Id::new("audio-model"), models, model, false) {
picked_model = Some(id);
}
if let Some(m) = model {
audio_toolbar(ui, &m.audio, settings);
}
},
(Slot::AbovePrompt, Some(m)) => reference_row(ui, m, refs, playing, image.as_ref(), &mut row_actions),
(Slot::BelowPrompt, Some(m)) if m.audio.style_prompt => style_row(ui, &mut settings.style_prompt),
_ => {},
};
let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots);
self.settings.prompt = prompt;
if let Some(id) = picked_model {
self.settings.model = Some(id);
}
if let Some(m) = model {
for action in row_actions {
self.row_action(action, env, m);
}
}
for action in actions {
self.handle(action, env, model);
}
}
fn handle(&mut self, action: PromptBoxAction, env: &mut Env<'_>, model: Option<&ModelInfo>) {
match action {
PromptBoxAction::Generate => self.generate(env, model),
// Toggles and tuning stay as they are.
PromptBoxAction::ClearAll => {
self.settings.prompt.clear();
self.settings.style_prompt.clear();
self.refs.clear();
},
PromptBoxAction::DroppedFiles(paths) => {
if let Some(m) = model {
self.add_dropped(env, m, &paths);
}
},
PromptBoxAction::PastedImage(png) => match model {
Some(m) if m.image_refs_max > 0 => {
self.make_room_for_image(env);
common::upload_png(env, &mut self.refs, png, ImageSlot::Reference);
},
_ => env.toasts.error("Image references aren't available here"),
},
PromptBoxAction::Deck(_) | PromptBoxAction::MentionPicked(..) => {},
}
}
fn row_action(&mut self, action: RowAction, env: &mut Env<'_>, m: &ModelInfo) {
match action {
RowAction::AddAudio => {
let paths = common::pick_files(&[RefKind::Audio], m.audio_refs_max > 1);
self.add_audio(env, m, &paths);
},
RowAction::AudioFromLibrary => {
let max = m.audio_refs_max.saturating_sub(self.refs.audios.len()).max(1);
env.requests.push(AppRequest::PickFromLibrary { kind: RefKind::Audio, slot: ImageSlot::Reference, max, page: MediaKind::Audio });
},
RowAction::AddImage => {
if let Some(path) = common::pick_files(&[RefKind::Image], false).first() {
self.make_room_for_image(env);
common::upload_path(env, &mut self.refs, path, RefKind::Image, ImageSlot::Reference);
}
},
RowAction::Remove(id) => {
if env.audio.is_playing(&ref_key(id)) {
env.audio.stop();
}
self.refs.remove(id);
},
RowAction::Toggle(id) => {
if let Some(url) = self.refs.audios.iter().find(|r| r.id == id).and_then(|r| r.full_url.clone()) {
env.requests.push(AppRequest::ToggleAudio { ref_id: id, url });
}
},
}
}
/// Dropped files (`usePromptBoxDrop`): audio wins over images; only the first image counts.
fn add_dropped(&mut self, env: &mut Env<'_>, m: &ModelInfo, paths: &[PathBuf]) {
let audios: Vec<PathBuf> = paths.iter().filter(|p| RefKind::from_path(p) == Some(RefKind::Audio)).cloned().collect();
let image = paths.iter().find(|p| RefKind::from_path(p) == Some(RefKind::Image));
if !audios.is_empty() {
self.add_audio(env, m, &audios);
} else if let Some(path) = image {
if m.image_refs_max == 0 {
env.toasts.error("Image references aren't available here");
return;
}
self.make_room_for_image(env);
common::upload_path(env, &mut self.refs, path, RefKind::Image, ImageSlot::Reference);
} else if !paths.is_empty() {
env.toasts.error(format!("Only {} files can be added here", accepted_kinds(m)));
}
}
/// Uploads audio tracks into the free slots (dropping an image reference: they don't mix).
fn add_audio(&mut self, env: &mut Env<'_>, m: &ModelInfo, paths: &[PathBuf]) {
if paths.is_empty() {
return;
}
if m.audio_refs_max == 0 {
env.toasts.error("Audio references aren't available here");
return;
}
let free = m.audio_refs_max.saturating_sub(self.refs.audios.len());
if paths.len() > free {
env.toasts.error(format!("Max {} audio track{}", m.audio_refs_max, if m.audio_refs_max == 1 { "" } else { "s" }));
}
if free == 0 {
return;
}
self.make_room_for_audio(env.toasts);
for path in paths.iter().take(free) {
common::upload_path(env, &mut self.refs, path, RefKind::Audio, ImageSlot::Reference);
}
}
/// Library picks, ready to attach: audio within the slots and the total-length cap.
pub fn attach_from_library(&mut self, toasts: &mut Toasts, catalog: &Catalog, kind: RefKind, picks: Vec<RefMedia>) {
let Some(m) = common::selected_model(&catalog.audio_page, self.settings.model.as_deref(), DEFAULT_AUDIO_MODEL) else {
return;
};
if kind != RefKind::Audio || picks.is_empty() {
common::attach_from_library(&mut self.refs, kind, ImageSlot::Reference, picks);
return;
}
self.make_room_for_audio(toasts);
let mut total = self.refs.total_secs(RefKind::Audio);
let mut accepted = Vec::new();
for pick in picks.into_iter().take(m.audio_refs_max.saturating_sub(self.refs.audios.len())) {
if total + pick.duration_secs > MAX_REF_SECS {
toasts.error(duration_message());
break;
}
total += pick.duration_secs;
accepted.push(pick);
}
common::attach_from_library(&mut self.refs, kind, ImageSlot::Reference, accepted);
}
/// After an upload lands: keep the audio references under the total-length cap.
pub fn check_durations(&mut self, toasts: &mut Toasts, ref_id: u64) {
if self.refs.audios.iter().any(|r| r.id == ref_id) && self.refs.total_secs(RefKind::Audio) > MAX_REF_SECS + 0.5 {
self.refs.remove(ref_id);
toasts.error(duration_message());
}
}
/// Audio and image references don't mix: adding audio drops the image.
fn make_room_for_audio(&mut self, toasts: &mut Toasts) {
if !self.refs.images.is_empty() {
self.refs.images.clear();
toasts.error("Removed image reference \u{2014} it can't be combined with audio");
}
}
/// One image at most, and never with audio.
fn make_room_for_image(&mut self, env: &mut Env<'_>) {
if !self.refs.audios.is_empty() {
if self.refs.audios.iter().any(|r| env.audio.is_playing(&ref_key(r.id))) {
env.audio.stop();
}
self.refs.audios.clear();
env.toasts.error("Removed audio reference \u{2014} it can't be combined with an image");
}
self.refs.images.clear();
}
fn missing_required_ref(&self, m: &ModelInfo) -> bool {
m.audio.requires_audio_ref && self.refs.audios.len() != 1
}
fn generate(&mut self, env: &mut Env<'_>, model: Option<&ModelInfo>) {
let Some(m) = model.filter(|_| !self.generating && !self.settings.prompt.trim().is_empty()) else {
return;
};
if self.missing_required_ref(m) {
env.toasts.error(format!("{} needs an audio track to work from \u{2014} add one first", m.name));
return;
}
if !env.signed_in {
env.requests.push(AppRequest::SignIn);
return;
}
if self.refs.any_uploading() {
env.toasts.error("Please wait for your references to finish uploading");
return;
}
self.generating = true;
let meta = EnqueueMeta { prompt: self.settings.prompt.trim().to_owned(), model_id: m.id.clone(), batch_count: 1, ref_image: None };
env.backend.generate(Modality::Audio, self.fields(m, false), meta);
}
pub fn on_enqueued(&mut self, env: &mut Env<'_>, job_tokens: &[String], meta: &EnqueueMeta) {
self.generating = false;
self.feed.add_enqueued(MediaKind::Audio, job_tokens, meta, env.catalog);
env.toasts.success("Audio generation enqueued!");
}
pub fn on_enqueue_failed(&mut self, env: &mut Env<'_>, message: &str, status: Option<u16>) {
self.generating = false;
env.toasts.error(failure_message(message, status));
}
/// The OmniGen request (`buildAudioGenerationBody`): only what the model takes, nothing null.
/// The estimate counts references with placeholders and needs no prompt.
fn fields(&self, m: &ModelInfo, estimate: bool) -> Map<String, Value> {
let (s, caps) = (&self.settings, &m.audio);
let mut f = Map::new();
f.insert("model".into(), json!(m.id));
let tokens = |list: &[RefMedia]| -> Vec<String> {
if estimate {
vec!["placeholder".to_owned(); list.len()]
} else {
list.iter().filter_map(|r| r.token.clone()).collect()
}
};
for (key, max, list) in [("audio_media_tokens", m.audio_refs_max, &self.refs.audios), ("image_media_tokens", m.image_refs_max, &self.refs.images)] {
let list = tokens(list.as_slice());
if max > 0 && !list.is_empty() {
f.insert(key.into(), json!(list));
}
}
if let Some(hz) = caps.sample_rate(s.sample_rate_hz).filter(|_| !caps.sample_rates.is_empty()) {
f.insert("sample_rate_hz".into(), json!(hz));
}
if estimate {
return f;
}
let (prompt, style) = (s.prompt.trim(), s.style_prompt.trim());
if !prompt.is_empty() {
f.insert("prompt".into(), json!(prompt));
}
if caps.style_prompt && !style.is_empty() {
f.insert("style_prompt".into(), json!(style));
}
for (supported, key, value) in [(caps.keep_lyrics, "keep_lyrics", s.keep_lyrics), (caps.instrumental, "is_instrumental", s.instrumental), (caps.loopable, "is_loopable", s.loopable)] {
if supported {
f.insert(key.into(), json!(value));
}
}
if let Some(bpm) = s.bpm.filter(|_| caps.bpm) {
f.insert("bpm".into(), json!(bpm));
}
if caps.musical_key {
f.insert("musical_key".into(), json!(musical_key(&s.musical_key).0));
}
for (supported, key, value) in [(caps.speed, "speed", s.speed), (caps.volume, "volume", s.volume)] {
if supported {
f.insert(key.into(), json!(hundredths(value)));
}
}
if caps.pitch {
f.insert("pitch".into(), json!(s.pitch));
}
f
}
}
/// The audio pickers after the model (`PromptBoxAudio`'s toolbar): the toggles the model offers,
/// Beat & Key, and Tuning.
fn audio_toolbar(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) {
let toggles = [(caps.instrumental, &mut s.instrumental, "audio-instrumental", Icon::MicOff, "Instrumental"), (caps.keep_lyrics, &mut s.keep_lyrics, "audio-keep-lyrics", Icon::Mic, "Keep lyrics"), (caps.loopable, &mut s.loopable, "audio-loop", Icon::Repeat, "Loop")];
for (supported, value, id, icon, label) in toggles {
if supported && pickers::toggle(ui, Id::new(id), icon, label, *value, &format!("{label}: {}", if *value { "ON" } else { "OFF" })) {
*value = !*value;
}
}
if caps.bpm || caps.musical_key {
beat_and_key(ui, caps, s);
}
if caps.has_tuning() {
tuning(ui, caps, s);
}
}
/// The Beat & Key popover: a BPM slider with Auto, and the musical key grid.
fn beat_and_key(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) {
let mut parts = Vec::new();
if caps.bpm {
parts.push(s.bpm.map_or_else(|| "Auto BPM".to_owned(), |b| format!("{b} BPM")));
}
if caps.musical_key {
parts.push(musical_key(&s.musical_key).2.to_owned());
}
let label = if parts.is_empty() { "Beat".to_owned() } else { parts.join(" \u{b7} ") };
pickers::popover_pill(ui, Id::new("audio-beat"), Leading::Icon(Icon::Drum), &label, "Beat & Key", |ui| {
ui.set_width(POPOVER_WIDTH);
widgets::menu_header(ui, "Beat & Key");
padded(ui, |ui| {
if caps.bpm {
ui.horizontal(|ui| {
setting_name(ui, "BPM");
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
setting_value(ui, &s.bpm.map_or_else(|| "Auto".to_owned(), |b| b.to_string()));
if s.bpm.is_some() && ui.add(egui::Button::new(egui::RichText::new("Auto").size(11.5).color(theme::ACCENT_INK)).frame(false)).on_hover_cursor(egui::CursorIcon::PointingHand).clicked() {
s.bpm = None;
}
});
});
let mut bpm = s.bpm.unwrap_or(DEFAULT_BPM);
if full_slider(ui, &mut bpm, BPM_RANGE, 1.0).changed() {
s.bpm = Some(bpm);
}
}
if caps.musical_key {
ui.add_space(6.0);
setting_name(ui, "Musical key");
ui.spacing_mut().item_spacing = vec2(4.0, 4.0);
let width = ui.available_width();
let (auto, keys) = MUSICAL_KEYS.split_first().expect("keys");
if choice_chip(ui, auto.2, s.musical_key == auto.0, width, auto.1).clicked() {
s.musical_key = auto.0.to_owned();
}
let cell = (width - 12.0) / 4.0;
for row in keys.chunks(4) {
ui.horizontal(|ui| {
for (value, label, short) in row {
if choice_chip(ui, short, s.musical_key == *value, cell, label).clicked() {
s.musical_key = (*value).to_owned();
}
}
});
}
}
});
});
}
/// The Tuning popover: sample rate chips and the speed, volume and pitch sliders.
fn tuning(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) {
pickers::popover_pill(ui, Id::new("audio-tuning"), Leading::Icon(Icon::SlidersHorizontal), "Tuning", "Tuning", |ui| {
ui.set_width(POPOVER_WIDTH);
widgets::menu_header(ui, "Tuning");
padded(ui, |ui| {
if !caps.sample_rates.is_empty() {
setting_name(ui, "Sample rate");
let current = caps.sample_rate(s.sample_rate_hz);
ui.horizontal_wrapped(|ui| {
ui.spacing_mut().item_spacing = vec2(4.0, 4.0);
for hz in &caps.sample_rates {
let label = models::sample_rate_label(*hz);
if choice_chip(ui, &label, current == Some(*hz), 72.0, &label).clicked() {
s.sample_rate_hz = Some(*hz);
}
}
});
ui.add_space(6.0);
}
if caps.speed {
labeled_slider(ui, "Speed", &format!("{:.2}\u{d7}", s.speed), &mut s.speed, SPEED_RANGE, 0.05);
}
if caps.volume {
labeled_slider(ui, "Volume", &format!("{:.2}\u{d7}", s.volume), &mut s.volume, VOLUME_RANGE, 0.05);
}
if caps.pitch {
labeled_slider(ui, "Pitch", &format!("{}{} st", if s.pitch > 0 { "+" } else { "" }, s.pitch), &mut s.pitch, PITCH_RANGE, 1.0);
}
});
});
}
/// The references above the prompt (`AudioReferenceRow`): the audio tracks (play, remove), add or
/// pick more while there's room, and the image when the model takes one.
fn reference_row(ui: &mut Ui, m: &ModelInfo, refs: &References, playing: Option<u64>, image: Option<&TextureHandle>, actions: &mut Vec<RowAction>) {
if m.audio_refs_max == 0 && m.image_refs_max == 0 {
return;
}
ui.horizontal_wrapped(|ui| {
ui.spacing_mut().item_spacing = vec2(8.0, 6.0);
if m.audio_refs_max > 0 {
icons::icon(ui, Icon::Music, 14.0, theme::MUTED);
ui.label(egui::RichText::new(format!("Audio Track ({}/{})", refs.audios.len(), m.audio_refs_max)).size(12.5).color(theme::MUTED));
if m.audio.requires_audio_ref && refs.audios.is_empty() {
ui.label(egui::RichText::new("required").size(12.5).color(theme::BAD));
}
for (i, r) in refs.audios.iter().enumerate() {
audio_tile(ui, i, r, playing == Some(r.id), actions);
}
if refs.audios.len() < m.audio_refs_max {
let uploading = refs.audios.iter().any(|r| r.status == RefStatus::Uploading);
if ui.add_enabled_ui(!uploading, |ui| widgets::pill(ui, Leading::Icon(Icon::Upload), if uploading { "Uploading\u{2026}" } else { "Add audio" }, false)).inner.clicked() {
actions.push(RowAction::AddAudio);
}
if widgets::pill(ui, Leading::Icon(Icon::Folder), "From library", false).clicked() {
actions.push(RowAction::AudioFromLibrary);
}
}
}
if m.image_refs_max > 0 {
ui.add_space(8.0);
ui.label(egui::RichText::new("Image").size(12.5).color(theme::MUTED));
match refs.images.first() {
Some(r) => image_tile(ui, r, image, actions),
None => {
if widgets::pill(ui, Leading::Icon(Icon::Image), "Add image", false).clicked() {
actions.push(RowAction::AddImage);
}
},
}
}
});
ui.add_space(8.0);
hairline(ui);
ui.add_space(8.0);
}
/// One audio reference (`AudioRefTile`): play / stop, "Audio 1 · 12s", remove.
fn audio_tile(ui: &mut Ui, index: usize, r: &RefMedia, playing: bool, actions: &mut Vec<RowAction>) {
let uploading = r.status == RefStatus::Uploading;
let text = if uploading { format!("Audio {}", index + 1) } else { format!("Audio {} \u{b7} {}s", index + 1, r.duration_secs.round() as i64) };
let galley = ui.painter().layout_no_wrap(text, FontId::new(12.5, theme::medium()), theme::INK);
let (rect, _) = ui.allocate_exact_size(vec2(6.0 + 24.0 + 8.0 + galley.size().x + 6.0 + 20.0 + 6.0, theme::CONTROL_H), Sense::hover());
ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, Stroke::new(1.0, theme::LINE), egui::StrokeKind::Inside);
let play = Rect::from_center_size(pos2(rect.left() + 18.0, rect.center().y), vec2(24.0, 24.0));
if uploading {
icons::paint_spinner(ui, play.shrink(4.0), theme::MUTED);
} else {
let resp = ui.interact(play, Id::new(("audio-ref-play", r.id)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
let label = if playing { "Stop" } else { "Play" };
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
let fill = match (playing, resp.hovered()) {
(true, _) => theme::ACCENT,
(false, true) => theme::WASH_HOVER,
(false, false) => theme::WASH,
};
ui.painter().circle_filled(play.center(), 12.0, fill);
if playing {
ui.painter().rect_filled(Rect::from_center_size(play.center(), vec2(8.0, 8.0)), 1.0, Color32::WHITE);
} else {
icons::paint(ui.painter(), play.shrink(7.0).translate(vec2(1.0, 0.0)), Icon::Play, theme::INK);
}
if resp.clicked() {
actions.push(RowAction::Toggle(r.id));
}
}
ui.painter().galley(pos2(play.right() + 8.0, rect.center().y - galley.size().y / 2.0), galley, theme::INK);
let remove = Rect::from_center_size(pos2(rect.right() - 16.0, rect.center().y), vec2(20.0, 20.0));
if x_button(ui, remove, Id::new(("audio-ref-remove", r.id)), "Remove audio", true) {
actions.push(RowAction::Remove(r.id));
}
}
/// The image reference: a 36 px thumbnail with a remove × on hover.
fn image_tile(ui: &mut Ui, r: &RefMedia, texture: Option<&TextureHandle>, actions: &mut Vec<RowAction>) {
let (rect, _) = ui.allocate_exact_size(vec2(36.0, 36.0), Sense::hover());
ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, Stroke::new(1.0, theme::LINE), egui::StrokeKind::Inside);
if let Some(t) = texture {
egui::Image::new(t).uv(cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, rect.shrink(1.0));
}
if r.status == RefStatus::Uploading {
icons::paint_spinner(ui, rect.shrink(9.0), Color32::WHITE);
}
let remove = Rect::from_min_size(pos2(rect.right() - 14.0, rect.top() - 4.0), vec2(18.0, 18.0));
let shown = ui.rect_contains_pointer(rect.union(remove));
if x_button(ui, remove, Id::new(("image-ref-remove", r.id)), "Remove image", shown) {
actions.push(RowAction::Remove(r.id));
}
}
/// The style line under the prompt (`StylePromptRow`).
fn style_row(ui: &mut Ui, style: &mut String) {
ui.add_space(6.0);
hairline(ui);
ui.add_space(6.0);
ui.horizontal(|ui| {
icons::icon(ui, Icon::Tags, 14.0, theme::DIM);
let clear_w = if style.is_empty() { 0.0 } else { 28.0 };
ui.add(egui::TextEdit::singleline(style).id_salt("audio-style").hint_text(STYLE_PLACEHOLDER).frame(egui::Frame::NONE).font(FontId::new(14.0, egui::FontFamily::Proportional)).desired_width(ui.available_width() - clear_w));
if !style.is_empty() {
let (rect, _) = ui.allocate_exact_size(vec2(20.0, 20.0), Sense::hover());
if x_button(ui, rect, Id::new("audio-style-clear"), "Clear style", true) {
style.clear();
}
}
});
}
/// A small round × at `rect`, painted while `shown` (it stays clickable for keyboards and tests).
fn x_button(ui: &mut Ui, rect: Rect, id: Id, label: &str, shown: bool) -> bool {
let resp = ui.interact(rect, id, Sense::click());
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
if shown || resp.hovered() {
let fill = if resp.hovered() { theme::fade(theme::DANGER, 0.7) } else { Color32::from_black_alpha(128) };
ui.painter().circle_filled(rect.center(), rect.width() / 2.0, fill);
icons::paint(ui.painter(), rect.shrink(rect.width() * 0.27), Icon::X, Color32::WHITE);
}
resp.on_hover_text(label).on_hover_cursor(egui::CursorIcon::PointingHand).clicked()
}
/// A selectable chip in the popovers (keys, sample rates): outlined, filled when selected.
fn choice_chip(ui: &mut Ui, label: &str, selected: bool, width: f32, tooltip: &str) -> Response {
let (rect, resp) = ui.allocate_exact_size(vec2(width, 28.0), Sense::click());
let (fill, border) = match (selected, resp.hovered()) {
(true, _) => (theme::WASH_HOVER, theme::fade(Color32::WHITE, 0.8)),
(false, true) => (theme::WASH, theme::LINE_STRONG),
(false, false) => (Color32::TRANSPARENT, theme::LINE),
};
ui.painter().rect(rect, theme::RADIUS, fill, Stroke::new(1.0, border), egui::StrokeKind::Inside);
ui.painter().text(rect.center(), Align2::CENTER_CENTER, label, FontId::new(12.5, theme::medium()), if selected { theme::INK } else { theme::MUTED });
resp.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Button, true, selected, label));
resp.on_hover_text(tooltip).on_hover_cursor(egui::CursorIcon::PointingHand)
}
/// A setting's name, its value at the right, and a slider across the popover.
fn labeled_slider<N: egui::emath::Numeric>(ui: &mut Ui, name: &str, value: &str, number: &mut N, range: RangeInclusive<N>, step: f64) {
ui.horizontal(|ui| {
setting_name(ui, name);
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| setting_value(ui, value));
});
full_slider(ui, number, range, step);
ui.add_space(4.0);
}
fn full_slider<N: egui::emath::Numeric>(ui: &mut Ui, number: &mut N, range: RangeInclusive<N>, step: f64) -> Response {
ui.style_mut().spacing.slider_width = ui.available_width();
ui.add(egui::Slider::new(number, range).step_by(step).show_value(false))
}
fn setting_name(ui: &mut Ui, name: &str) {
ui.label(egui::RichText::new(name).size(12.0).color(theme::MUTED));
}
fn setting_value(ui: &mut Ui, value: &str) {
ui.label(egui::RichText::new(value).size(12.0).family(theme::semibold()).color(theme::INK));
}
fn padded(ui: &mut Ui, content: impl FnOnce(&mut Ui)) {
egui::Frame::new().inner_margin(egui::Margin::symmetric(8, 6)).show(ui, content);
}
fn hairline(ui: &mut Ui) {
let (line, _) = ui.allocate_exact_size(vec2(ui.available_width(), 1.0), Sense::hover());
ui.painter().rect_filled(line, 0.0, theme::fade(Color32::WHITE, 0.1));
}
/// The key a stored value names (Auto for anything unknown): value, label, short label.
fn musical_key(value: &str) -> &'static (&'static str, &'static str, &'static str) {
MUSICAL_KEYS.iter().find(|k| k.0 == value).unwrap_or(&MUSICAL_KEYS[0])
}
/// The audio player's key for a reference clip.
fn ref_key(id: u64) -> String {
format!("ref:{id}")
}
/// A slider's value to two decimals (0.05 steps drift in `f32`).
fn hundredths(value: f32) -> f64 {
(f64::from(value) * 100.0).round() / 100.0
}
/// "audio", "audio or image" (for the drop toasts).
fn accepted_kinds(m: &ModelInfo) -> String {
let kinds: Vec<&str> = [("audio", m.audio_refs_max), ("image", m.image_refs_max)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect();
if kinds.is_empty() {
"reference".to_owned()
} else {
kinds.join(" or ")
}
}
fn duration_message() -> String {
format!("Total audio duration cannot exceed {}s", MAX_REF_SECS as u32)
}
/// What a failed generate tells the user: out of credits, an outage, or the server's message.
fn failure_message(message: &str, status: Option<u16>) -> String {
match status {
Some(402) => "Not enough credits for this generation".to_owned(),
Some(code) if code >= 500 => OUTAGE_MESSAGE.to_owned(),
_ if message.is_empty() => "Failed to start audio generation".to_owned(),
_ => message.to_owned(),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn model(json: Value) -> ModelInfo {
models::audio_model(&serde_json::from_value(json).expect("audio model"))
}
/// The server's audio models (`configs/omni_gen/audio_models.rs`).
fn suno_music() -> ModelInfo {
model(json!({ "model": "suno_music", "style_prompt_supported": true, "instrumental_toggle_supported": true }))
}
fn suno_sounds() -> ModelInfo {
model(json!({ "model": "suno_sounds", "loopable_toggle_supported": true, "bpm_supported": true, "musical_key_supported": true }))
}
fn suno_remix() -> ModelInfo {
model(json!({ "model": "suno_remix", "style_prompt_supported": true, "keep_lyrics_supported": true, "audio_references_supported": true, "audio_references_max": 1 }))
}
fn seed_audio() -> ModelInfo {
model(json!({ "model": "seed_audio_1p0", "audio_references_supported": true, "audio_references_max": 3, "image_references_supported": true, "image_references_max": 1, "sample_rate_hz_options": [8000, 16000, 24000, 32000, 44100, 48000], "sample_rate_hz_default": 24000, "speed_supported": true, "volume_supported": true, "pitch_supported": true }))
}
fn page(settings: AudioSettings) -> CreateAudioPage {
CreateAudioPage { settings, ..Default::default() }
}
fn audio_ref(token: &str, secs: f32) -> RefMedia {
RefMedia::from_library(RefKind::Audio, token.into(), None, Some(format!("https://cdn/{token}.mp3")), secs)
}
mod request_bodies {
use super::*;
#[test]
fn suno_music_sends_only_what_it_takes() {
let p = page(AudioSettings { prompt: " lofi beat ".into(), style_prompt: " chill ".into(), instrumental: true, keep_lyrics: true, bpm: Some(90), ..Default::default() });
assert_eq!(Value::Object(p.fields(&suno_music(), false)), json!({ "model": "suno_music", "prompt": "lofi beat", "style_prompt": "chill", "is_instrumental": true }));
}
#[test]
fn suno_sounds_sends_its_toggles_beat_and_key() {
let mut p = page(AudioSettings { prompt: "door creak".into(), ..Default::default() });
assert_eq!(Value::Object(p.fields(&suno_sounds(), false)), json!({ "model": "suno_sounds", "prompt": "door creak", "is_loopable": false, "musical_key": "auto" }), "Auto BPM is left to the model");
p.settings.bpm = Some(128);
p.settings.musical_key = "c_minor".into();
let f = p.fields(&suno_sounds(), false);
assert_eq!((f["bpm"].clone(), f["musical_key"].clone()), (json!(128), json!("c_minor")));
}
#[test]
fn seed_audio_sends_references_and_tuning() {
let mut p = page(AudioSettings { prompt: "rain".into(), speed: 1.05, volume: 0.5, pitch: -3, sample_rate_hz: Some(11025), ..Default::default() });
p.refs.audios = vec![audio_ref("mf_a", 10.0), audio_ref("mf_b", 20.0)];
assert_eq!(Value::Object(p.fields(&seed_audio(), false)), json!({ "model": "seed_audio_1p0", "prompt": "rain", "audio_media_tokens": ["mf_a", "mf_b"], "sample_rate_hz": 24000, "speed": 1.05, "volume": 0.5, "pitch": -3 }));
}
#[test]
fn estimates_count_references_without_the_prompt() {
let mut p = page(AudioSettings { prompt: "rain".into(), sample_rate_hz: Some(44100), ..Default::default() });
p.refs.audios = vec![RefMedia::uploading(RefKind::Audio, None), audio_ref("mf_b", 5.0)];
assert_eq!(Value::Object(p.fields(&seed_audio(), true)), json!({ "model": "seed_audio_1p0", "audio_media_tokens": ["placeholder", "placeholder"], "sample_rate_hz": 44100 }));
assert_eq!(Value::Object(p.fields(&suno_music(), true)), json!({ "model": "suno_music" }), "references the model can't take don't count");
}
}
mod rules {
use super::*;
#[test]
fn remixes_need_exactly_one_track() {
let mut p = page(AudioSettings::default());
assert!(p.missing_required_ref(&suno_remix()));
assert!(!p.missing_required_ref(&suno_music()));
p.refs.audios.push(audio_ref("mf_a", 30.0));
assert!(!p.missing_required_ref(&suno_remix()));
p.refs.audios.push(audio_ref("mf_b", 30.0));
assert!(p.missing_required_ref(&suno_remix()), "two tracks are too many");
}
#[test]
fn library_picks_respect_slots_the_cap_and_drop_the_image() {
let mut catalog = Catalog::default();
catalog.set_audio(vec![seed_audio()]);
let mut p = page(AudioSettings { model: Some("seed_audio_1p0".into()), ..Default::default() });
p.refs.images.push(RefMedia::from_library(RefKind::Image, "mf_img".into(), None, None, 0.0));
let mut toasts = Toasts::default();
p.attach_from_library(&mut toasts, &catalog, RefKind::Audio, vec![audio_ref("mf_a", 400.0), audio_ref("mf_b", 300.0), audio_ref("mf_c", 10.0)]);
let tokens: Vec<_> = p.refs.audios.iter().filter_map(|r| r.token.as_deref()).collect();
assert_eq!(tokens, ["mf_a"], "the second would pass 600s, so picking stops there");
assert!(p.refs.images.is_empty(), "audio and an image don't mix");
}
#[test]
fn uploads_past_the_cap_are_dropped() {
let mut p = page(AudioSettings::default());
p.refs.audios = vec![audio_ref("mf_a", 590.0), audio_ref("mf_b", 30.0)];
let late = p.refs.audios[1].id;
p.check_durations(&mut Toasts::default(), late);
assert_eq!(p.refs.audios.len(), 1);
}
#[test]
fn failures_explain_credits_and_outages() {
assert_eq!(failure_message("Payment required", Some(402)), "Not enough credits for this generation");
assert_eq!(failure_message("boom", Some(503)), OUTAGE_MESSAGE);
assert_eq!(failure_message("Prompt too long", Some(400)), "Prompt too long");
assert_eq!(failure_message("", None), "Failed to start audio generation");
}
}
/// The toolbar headlessly (egui_kittest): toggles, Beat & Key and Tuning.
mod toolbar {
use egui_kittest::Harness;
use egui_kittest::kittest::{NodeT, Queryable};
use super::*;
struct State {
caps: AudioCaps,
settings: AudioSettings,
fonts_ready: bool,
}
#[test]
fn toggles_show_per_model_and_flip() {
let mut harness = build(suno_music().audio);
assert!(harness.query_by_label("Loop").is_none(), "Suno Music has no loop toggle");
harness.get_by_label("Instrumental").click();
harness.run();
assert!(harness.state().settings.instrumental);
}
#[test]
fn beat_and_key_sets_the_key_and_bpm() {
let mut harness = build(suno_sounds().audio);
harness.get_by_label("Auto BPM \u{b7} Auto").click();
harness.run();
harness.get_by_label("Cm").click();
harness.run();
assert_eq!(harness.state().settings.musical_key, "c_minor");
assert!(harness.query_by_label("Auto BPM \u{b7} Cm").is_some(), "the pill shows the key");
}
#[test]
fn tuning_picks_a_sample_rate() {
let mut harness = build(seed_audio().audio);
harness.get_by_label("Tuning").click();
harness.run();
// egui reports a selected button as toggled.
assert!(harness.query_by_label("24 kHz").is_some_and(|n| n.accesskit_node().toggled() == Some(egui::accesskit::Toggled::True)), "the default is selected");
harness.get_by_label("44.1 kHz").click();
harness.run();
assert_eq!(harness.state().settings.sample_rate_hz, Some(44100));
}
fn build(caps: AudioCaps) -> Harness<'static, State> {
let mut harness = Harness::builder().with_size(egui::vec2(1000.0, 600.0)).build_ui_state(
|ui, s: &mut State| {
if !s.fonts_ready {
crate::theme::apply(ui.ctx());
s.fonts_ready = true;
return;
}
ui.add_space(400.0);
ui.horizontal(|ui| audio_toolbar(ui, &s.caps, &mut s.settings));
},
State { caps, settings: AudioSettings::default(), fonts_ready: false },
);
harness.run();
harness
}
}
}
+32 -28
View File
@@ -15,7 +15,7 @@ use crate::pages::toolbar;
use crate::prompt_box::deck::{DeckAction, DeckLimits};
use crate::prompt_box::model_selector::model_selector;
use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, References};
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, editor};
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot, editor};
const PLACEHOLDER: &str = "Describe what you want in the image...";
@@ -69,38 +69,41 @@ impl CreateImagePage {
let deck = if max_images > 0 { DeckMode::References(DeckLimits { max_images, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: env.signed_in }) } else { DeckMode::None };
let accepts: &[RefKind] = if max_images > 0 { &[RefKind::Image] } else { &[] };
let prompt_max = model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max);
let props = PromptBoxProps { id: Id::new("image-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: Some(prompt_max), deck, accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some(), generating: self.generating, generate_tooltip: "Generate", warning: None, banner: None };
let props = PromptBoxProps { id: Id::new("image-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: Some(prompt_max), deck, accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some(), generating: self.generating, generate_tooltip: "Generate", warning: None, banner: None, extra_input: false };
let models = &env.catalog.image_page;
let settings = &mut self.settings;
let mut picked_model = None;
let mut left = |ui: &mut Ui| {
if let Some(id) = model_selector(ui, Id::new("image-model"), models, model, false) {
picked_model = Some(id);
}
let Some(m) = model else {
return;
};
let aspect = ModelInfo::resolve(settings.aspect_ratio.as_deref(), &m.aspect_ratios, m.aspect_default.as_ref());
if let Some(v) = toolbar::aspect_ratio(ui, Id::new("image-aspect"), m, aspect.as_deref().unwrap_or("")) {
settings.aspect_ratio = Some(v);
}
let resolution = ModelInfo::resolve(settings.resolution.as_deref(), &m.resolutions, m.resolution_default.as_ref());
if let Some(v) = toolbar::resolution(ui, Id::new("image-resolution"), m, resolution.as_deref().unwrap_or("")) {
settings.resolution = Some(v);
}
let quality = ModelInfo::resolve(settings.quality.as_deref(), &m.qualities, m.quality_default.as_ref());
if let Some(v) = toolbar::quality(ui, Id::new("image-quality"), m, quality.as_deref().unwrap_or("")) {
settings.quality = Some(v);
}
};
let count = settings.count;
let mut picked_model = None;
let mut picked_count = None;
let mut right = |ui: &mut Ui| {
if let Some(m) = model {
picked_count = toolbar::count(ui, Id::new("image-count"), m, count, "images");
}
let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) {
(Slot::ToolbarLeft, _) => {
if let Some(id) = model_selector(ui, Id::new("image-model"), models, model, false) {
picked_model = Some(id);
}
let Some(m) = model else {
return;
};
let aspect = ModelInfo::resolve(settings.aspect_ratio.as_deref(), &m.aspect_ratios, m.aspect_default.as_ref());
if let Some(v) = toolbar::aspect_ratio(ui, Id::new("image-aspect"), m, aspect.as_deref().unwrap_or("")) {
settings.aspect_ratio = Some(v);
}
let resolution = ModelInfo::resolve(settings.resolution.as_deref(), &m.resolutions, m.resolution_default.as_ref());
if let Some(v) = toolbar::resolution(ui, Id::new("image-resolution"), m, resolution.as_deref().unwrap_or("")) {
settings.resolution = Some(v);
}
let quality = ModelInfo::resolve(settings.quality.as_deref(), &m.qualities, m.quality_default.as_ref());
if let Some(v) = toolbar::quality(ui, Id::new("image-quality"), m, quality.as_deref().unwrap_or("")) {
settings.quality = Some(v);
}
},
(Slot::ToolbarRight, Some(m)) => {
if let Some(n) = toolbar::count(ui, Id::new("image-count"), m, count, "images") {
picked_count = Some(n);
}
},
_ => {},
};
let actions = prompt_box::show(ui, &mut settings.prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut left, &mut right);
let actions = prompt_box::show(ui, &mut settings.prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots);
if let Some(id) = picked_model {
self.settings.model = Some(id);
}
@@ -143,6 +146,7 @@ impl CreateImagePage {
DeckAction::SwapFrames => {},
},
PromptBoxAction::DroppedFiles(paths) => self.add_dropped(env, &paths, max_images),
PromptBoxAction::MentionPicked(..) => {},
PromptBoxAction::PastedImage(png) => {
if self.refs.images.len() >= max_images {
env.toasts.error(max_images_message(max_images));
+120 -22
View File
@@ -20,7 +20,7 @@ use crate::prompt_box::editor::{self, MentionItem, MentionKind};
use crate::prompt_box::model_selector::model_selector;
use crate::prompt_box::pickers::{self, PickOption, Trigger};
use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, References};
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState};
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot};
use crate::ui::icons::Icon;
use crate::ui::widgets::Leading;
@@ -67,6 +67,8 @@ pub struct CreateVideoPage {
pub generating: bool,
box_state: PromptBoxState,
box_height: f32,
/// Which character each mentioned name means, when the user picked one explicitly.
character_picks: std::collections::HashMap<String, String>,
}
impl CreateVideoPage {
@@ -104,10 +106,14 @@ impl CreateVideoPage {
(DeckMode::References(l), _) => [(RefKind::Image, l.max_images), (RefKind::Video, l.max_videos), (RefKind::Audio, l.max_audios)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect(),
_ => Vec::new(),
};
let mentions = if reference { self.mentions() } else { Vec::new() };
let characters_on = model.is_some_and(supports_characters);
let mut mentions = if reference { self.mentions() } else { Vec::new() };
if characters_on {
mentions.extend(editor::character_mentions(env.characters.iter().map(|c| (c.name.as_str(), c.token.as_str(), c.avatar_url()))));
}
let needs_start = model.is_some_and(|m| self.needs_start_frame(m));
let banner = model.filter(|m| !m.text_to_video && !self.has_image_input()).map(|_| TEXT_ONLY_BANNER);
let props = PromptBoxProps { id: Id::new("video-prompt-box"), placeholder: if reference { PLACEHOLDER_REFERENCE } else { PLACEHOLDER }, enter_to_generate: env.enter_to_generate, mentions: &mentions, max_length: Some(model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max)), deck, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !needs_start, generating: self.generating, generate_tooltip: if needs_start { "Add a starting image before generating" } else { "Generate" }, warning: needs_start.then_some("Starting frame required"), banner };
let props = PromptBoxProps { id: Id::new("video-prompt-box"), placeholder: if reference { PLACEHOLDER_REFERENCE } else { PLACEHOLDER }, enter_to_generate: env.enter_to_generate, mentions: &mentions, max_length: Some(model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max)), deck, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !needs_start, generating: self.generating, generate_tooltip: if needs_start { "Add a starting image before generating" } else { "Generate" }, warning: needs_start.then_some("Starting frame required"), banner, extra_input: false };
let models = &env.catalog.video_page;
let mut prompt = std::mem::take(&mut self.settings.prompt);
@@ -116,21 +122,28 @@ impl CreateVideoPage {
let with_refs = reference && !self.refs.images.is_empty();
let mut picked_model = None;
let mut picked_mode = None;
let mut left = |ui: &mut Ui| {
if let Some(id) = model_selector(ui, Id::new("video-model"), models, model, true) {
picked_model = Some(id);
}
if let Some(m) = model {
picked_mode = video_toolbar(ui, m, settings, with_refs).or(picked_mode);
}
};
let mut picked_count = None;
let mut right = |ui: &mut Ui| {
if let Some(m) = model {
picked_count = toolbar::count(ui, Id::new("video-count"), m, count, "videos");
}
let mut open_characters = false;
let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) {
(Slot::ToolbarLeft, _) => {
if let Some(id) = model_selector(ui, Id::new("video-model"), models, model, true) {
picked_model = Some(id);
}
if let Some(m) = model {
picked_mode = video_toolbar(ui, m, settings, with_refs).or(picked_mode);
if characters_on && pickers::toggle(ui, Id::new("video-characters"), Icon::User, "@Characters", false, "Characters") {
open_characters = true;
}
}
},
(Slot::ToolbarRight, Some(m)) => {
if let Some(n) = toolbar::count(ui, Id::new("video-count"), m, count, "videos") {
picked_count = Some(n);
}
},
_ => {},
};
let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut left, &mut right);
let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots);
self.settings.prompt = prompt;
if let Some(id) = picked_model {
self.settings.model = Some(id);
@@ -141,6 +154,9 @@ impl CreateVideoPage {
if let Some(mode) = picked_mode {
switch_mode(&mut self.settings, &mut self.refs, mode);
}
if open_characters {
env.requests.push(AppRequest::OpenCharacters);
}
for action in actions {
self.handle(action, env, model, limits.as_ref());
}
@@ -176,14 +192,27 @@ impl CreateVideoPage {
let item = self.refs.images.remove(from);
self.refs.images.insert(to.min(self.refs.images.len()), item);
},
DeckAction::Preview(id) => {
if let Some(url) = self.refs.find_mut(id).and_then(|r| r.full_url.clone().or_else(|| r.preview.clone())) {
env.requests.push(AppRequest::Preview(url));
}
DeckAction::Preview(id) => match self.refs.find_mut(id) {
// Audio cards play and stop on click (the webapp's DeckCard).
Some(r) if r.kind == RefKind::Audio => {
if let Some(url) = r.full_url.clone() {
env.requests.push(AppRequest::ToggleAudio { ref_id: id, url });
}
},
Some(r) => {
if let Some(url) = r.full_url.clone().or_else(|| r.preview.clone()) {
env.requests.push(if r.kind == RefKind::Video { AppRequest::PlayExternally(url) } else { AppRequest::Preview(url) });
}
},
None => {},
},
DeckAction::SwapFrames => self.refs.swap_frames(),
},
PromptBoxAction::DroppedFiles(paths) => self.add_files(env, &paths, ImageSlot::Reference, limits),
PromptBoxAction::MentionPicked(label, Some(token)) => {
self.character_picks.insert(label.trim_start_matches('@').to_owned(), token);
},
PromptBoxAction::MentionPicked(_, None) => {},
PromptBoxAction::PastedImage(png) => match self.settings.input_mode {
InputMode::Keyframe => match self.free_keyframe(model) {
Some(slot) => common::upload_png(env, &mut self.refs, png, slot),
@@ -288,7 +317,27 @@ impl CreateVideoPage {
self.generating = true;
let ref_image = self.refs.first_frame.as_ref().or(self.refs.images.first()).and_then(|r| r.preview.clone());
let meta = EnqueueMeta { prompt: self.settings.prompt.clone(), model_id: model.id.clone(), batch_count: u32::from(model.valid_batch(self.settings.count)), ref_image };
env.backend.generate(Modality::Video, self.fields(model, false), meta);
let mut fields = self.fields(model, false);
if supports_characters(model) {
let tokens = mentioned_characters(&self.settings.prompt, env.characters, &self.character_picks);
if !tokens.is_empty() {
fields.insert("reference_character_tokens".into(), json!(tokens));
}
}
env.backend.generate(Modality::Video, fields, meta);
}
/// The characters dialog picked `name`: mention it at the end of the prompt.
pub fn mention_character(&mut self, name: &str, token: &str) {
let p = &mut self.settings.prompt;
if !p.is_empty() && !p.ends_with(char::is_whitespace) {
p.push(' ');
}
p.push('@');
p.push_str(name);
p.push(' ');
self.character_picks.insert(name.to_owned(), token.to_owned());
self.box_state.editor.focus_requested = true;
}
/// The OmniGen request (`GenerateVideoRequest` after `api_fields`).
@@ -433,7 +482,7 @@ impl CreateVideoPage {
"audioref" => RefKind::Audio,
_ => RefKind::Image,
};
let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256), Some(c.media_links.cdn_url.clone()), 0.0);
let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256).filter(|_| kind != RefKind::Audio), Some(c.media_links.cdn_url.clone()), 0.0);
match (c.semantic.as_str(), kind) {
("vid_end_frame", _) => self.refs.last_frame = Some(item),
(_, RefKind::Image) if s.input_mode == InputMode::Keyframe && self.refs.first_frame.is_none() => self.refs.first_frame = Some(item),
@@ -506,6 +555,33 @@ fn switch_mode(s: &mut VideoSettings, refs: &mut References, mode: InputMode) {
s.input_mode = mode;
}
/// Whether the model takes `@Character` mentions (the listing says so; Seedance 2.0 always does).
fn supports_characters(m: &ModelInfo) -> bool {
m.character_refs_max > 0 || m.id == "seedance_2p0"
}
/// One token per character named in the prompt (`@Name`, not inside a longer word): the user's
/// explicit pick for that name, else the newest character with it.
fn mentioned_characters(prompt: &str, characters: &[crate::backend::wire::Character], picks: &std::collections::HashMap<String, String>) -> Vec<String> {
let mut names: Vec<&str> = characters.iter().map(|c| c.name.as_str()).filter(|n| !n.is_empty()).collect();
names.sort_by_key(|n| std::cmp::Reverse(n.len()));
names.dedup();
let mut tokens = Vec::new();
for name in names {
let needle = format!("@{name}");
let mentioned = prompt.match_indices(&needle).any(|(i, _)| !prompt[i + needle.len()..].chars().next().is_some_and(|c| c.is_alphanumeric() || c == '_'));
if !mentioned {
continue;
}
let candidates: Vec<&crate::backend::wire::Character> = characters.iter().filter(|c| c.name == name).collect();
let chosen = picks.get(name).and_then(|t| candidates.iter().find(|c| &c.token == t)).or(candidates.first());
if let Some(c) = chosen {
tokens.push(c.token.clone());
}
}
tokens
}
fn deck_limits(m: &ModelInfo, library: bool) -> DeckLimits {
DeckLimits { max_images: m.image_refs_max, max_videos: m.video_refs_max, max_video_secs: m.video_refs_max_secs.map(f32::from), max_audios: m.audio_refs_max, max_audio_secs: m.audio_refs_max_secs.map(f32::from), library }
}
@@ -539,3 +615,25 @@ fn frames_full_message(model: Option<&ModelInfo>) -> &'static str {
"The first frame is already set"
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::wire::Character;
fn character(token: &str, name: &str) -> Character {
Character { token: token.into(), name: name.into(), ..Default::default() }
}
#[test]
fn mentions_resolve_to_one_token_per_name() {
// Newest first, as the server lists them.
let characters = [character("c3", "Bob"), character("c2", "Bob2"), character("c1", "Bob"), character("c0", "Ann")];
let none = std::collections::HashMap::new();
assert_eq!(mentioned_characters("@Bob2 waves", &characters, &none), ["c2"], "@Bob2 isn't @Bob");
assert_eq!(mentioned_characters("@Bob, then @Ann.", &characters, &none), ["c3", "c0"], "newest Bob by default");
let picks = std::collections::HashMap::from([("Bob".to_owned(), "c1".to_owned())]);
assert_eq!(mentioned_characters("hi @Bob", &characters, &picks), ["c1"], "the explicit pick wins");
assert!(mentioned_characters("no mentions", &characters, &none).is_empty());
}
}
+1
View File
@@ -1,4 +1,5 @@
pub mod common;
pub mod create_audio;
pub mod create_image;
pub mod create_video;
pub mod other;
+6 -5
View File
@@ -23,12 +23,13 @@ pub fn home(ui: &mut Ui, signed_in: bool, requests: &mut Vec<AppRequest>) -> Opt
ui.add_space(10.0);
ui.label(widgets::heading("Capable tools for artists.", 52.0));
ui.add_space(10.0);
ui.label(egui::RichText::new("Generate images and videos with the best models, steer them with references and keyframes, and keep everything in your library.").size(17.0).color(theme::MUTED));
ui.label(egui::RichText::new("Generate images, videos and audio with the best models, steer them with references and keyframes, and keep everything in your library.").size(17.0).color(theme::MUTED));
ui.add_space(28.0);
ui.horizontal(|ui| {
ui.spacing_mut().item_spacing.x = 16.0;
let card_w = (width - 16.0) / 2.0;
for (page, icon, title, text) in [(Page::CreateImage, Icon::Image, "Create Image", "Describe anything. See it in seconds."), (Page::CreateVideo, Icon::Video, "Create Video", "Describe a scene. See it in motion.")] {
let cards = [(Page::CreateImage, Icon::Image, "Create Image", "Describe anything. See it in seconds."), (Page::CreateVideo, Icon::Video, "Create Video", "Describe a scene. See it in motion."), (Page::CreateAudio, Icon::Music, "Create Audio", "Describe a song, a sound, or a sample.")];
let card_w = (width - 16.0 * (cards.len() - 1) as f32) / cards.len() as f32;
for (page, icon, title, text) in cards {
if card(ui, card_w, icon, title, text) {
open = Some(page);
}
@@ -73,9 +74,9 @@ pub fn library(ui: &mut Ui, env: &mut Env<'_>, feed: &mut FeedStore) {
placeholder(ui, area, "Library", "Your generations will appear here.", None);
return;
}
let view = FeedView { id: Id::new("library-feed"), mode: env.view_mode, pending: &[], failed: &[], items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: true, bottom_padding: 24.0 };
let view = FeedView { id: Id::new("library-feed"), mode: env.view_mode, pending: &[], failed: &[], items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: true, autoplay: env.autoplay, bottom_padding: 24.0 };
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(area.shrink2(vec2(12.0, 0.0)).with_min_y(area.top() + 2.0)));
for action in grid::show(&mut child, &view, env.cache, env.ratios, env.catalog) {
for action in grid::show(&mut child, &view, env.cache, env.ratios, env.audio, env.catalog) {
match action {
FeedAction::Open(token) => {
let kind = feed.find(&token).map_or(MediaKind::Image, |i| i.kind);
+15 -2
View File
@@ -309,12 +309,14 @@ fn paint_card(ui: &Ui, painter: &egui::Painter, rect: Rect, tilt: f32, item: &Re
image.paint_at(ui, rect.shrink(2.0));
},
None => {
let playing = item.kind == RefKind::Audio && playing_audio(ui) == Some(item.id);
let icon = match item.kind {
RefKind::Image => Icon::Image,
RefKind::Video => Icon::Video,
RefKind::Audio => Icon::Music,
RefKind::Audio if playing => Icon::Volume,
RefKind::Audio => Icon::Play,
};
icons::paint(painter, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), icon, theme::MUTED);
icons::paint(painter, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), icon, if playing { theme::ACCENT_INK } else { theme::MUTED });
},
}
if item.kind != RefKind::Image && tilt == 0.0 {
@@ -334,6 +336,17 @@ fn paint_card(ui: &Ui, painter: &egui::Painter, rect: Rect, tilt: f32, item: &Re
painter.add(Shape::closed_line(corners.to_vec(), Stroke::new(2.0, border)));
}
/// The audio reference playing right now (the app sets it each frame).
pub fn set_playing_audio(ctx: &egui::Context, id: Option<u64>) {
ctx.data_mut(|d| d.insert_temp(Id::new(PLAYING_AUDIO), id));
}
fn playing_audio(ui: &Ui) -> Option<u64> {
ui.data(|d| d.get_temp::<Option<u64>>(Id::new(PLAYING_AUDIO))).flatten()
}
const PLAYING_AUDIO: &str = "deck-playing-audio";
/// The hover-revealed round × in a card's top-right corner. Returns whether it was clicked.
fn remove_button(ui: &mut Ui, id: Id, card: Rect, card_hovered: bool) -> bool {
let rect = Rect::from_min_size(pos2(card.right() - 22.0, card.top() + 2.0), vec2(20.0, 20.0));
+33 -15
View File
@@ -22,12 +22,15 @@ const RIGHT_GUTTER: f32 = 32.0;
const IMAGE_COLORS: [Color32; 5] = [Color32::from_rgb(96, 165, 250), Color32::from_rgb(251, 146, 60), Color32::from_rgb(167, 139, 250), Color32::from_rgb(52, 211, 153), Color32::from_rgb(251, 113, 133)];
const VIDEO_COLORS: [Color32; 3] = [Color32::from_rgb(250, 204, 21), Color32::from_rgb(245, 158, 11), Color32::from_rgb(74, 222, 128)];
const AUDIO_COLORS: [Color32; 2] = [Color32::from_rgb(192, 132, 252), Color32::from_rgb(232, 121, 249)];
/// Teal, emerald, sky.
const CHARACTER_COLORS: [Color32; 3] = [Color32::from_rgb(45, 212, 191), Color32::from_rgb(34, 197, 94), Color32::from_rgb(14, 165, 233)];
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MentionKind {
Image,
Video,
Audio,
Character,
}
/// Something the prompt can reference by name, e.g. `@Image2`.
@@ -39,6 +42,8 @@ pub struct MentionItem {
pub index: usize,
/// Media cache key of a thumbnail, if there is one.
pub preview: Option<String>,
/// The character this mention names (several can share a name).
pub token: Option<String>,
}
impl MentionItem {
@@ -48,6 +53,7 @@ impl MentionItem {
MentionKind::Image => IMAGE_COLORS[i % IMAGE_COLORS.len()],
MentionKind::Video => VIDEO_COLORS[i % VIDEO_COLORS.len()],
MentionKind::Audio => AUDIO_COLORS[i % AUDIO_COLORS.len()],
MentionKind::Character => CHARACTER_COLORS[i % CHARACTER_COLORS.len()],
}
}
}
@@ -93,6 +99,8 @@ pub struct EditorOutput {
pub submit: bool,
pub open_fullscreen: bool,
pub focused: bool,
/// A mention the user picked from the dropdown: (label, character token).
pub picked: Option<(String, Option<String>)>,
}
/// Draws the editor in the available width.
@@ -105,7 +113,7 @@ pub fn show(ui: &mut Ui, prompt: &mut String, state: &mut EditorState, opts: &Ed
// Keys the field mustn't see: dropdown navigation, and Enter when it generates.
if had_focus {
if !filtered.is_empty() {
handle_dropdown_keys(ui, id, prompt, state, &filtered);
out.picked = handle_dropdown_keys(ui, id, prompt, state, &filtered);
} else if opts.enter_to_generate && consume_plain_enter(ui) {
out.submit = true;
}
@@ -183,7 +191,9 @@ pub fn show(ui: &mut Ui, prompt: &mut String, state: &mut EditorState, opts: &Ed
}
if !filtered.is_empty() && out.focused {
mention_dropdown(ui, id, field_rect, prompt, state, &filtered, thumbnails);
if let Some(picked) = mention_dropdown(ui, id, field_rect, prompt, state, &filtered, thumbnails) {
out.picked = Some(picked);
}
}
out
}
@@ -214,12 +224,10 @@ fn consume_plain_enter(ui: &Ui) -> bool {
})
}
fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem]) {
fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem]) -> Option<(String, Option<String>)> {
let n = filtered.len();
let (down, up, accept, escape) = ui.input_mut(|i| (i.consume_key(Modifiers::NONE, Key::ArrowDown), i.consume_key(Modifiers::NONE, Key::ArrowUp), i.consume_key(Modifiers::NONE, Key::Enter) || i.consume_key(Modifiers::NONE, Key::Tab), i.consume_key(Modifiers::NONE, Key::Escape)));
let Some(query) = state.mention.as_mut() else {
return;
};
let query = state.mention.as_mut()?;
if down {
query.selected = (query.selected + 1) % n;
}
@@ -228,12 +236,14 @@ fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut Editor
}
if escape {
state.mention = None;
return;
return None;
}
if accept {
let label = filtered[query.selected.min(n - 1)].label.clone();
insert_mention(ui.ctx(), id, prompt, state, &label);
if !accept {
return None;
}
let item = filtered[query.selected.min(n - 1)];
insert_mention(ui.ctx(), id, prompt, state, &item.label);
Some((item.label.clone(), item.token.clone()))
}
/// Opens, updates or closes the dropdown from the text before the cursor.
@@ -284,7 +294,7 @@ fn insert_mention(ctx: &egui::Context, id: Id, prompt: &mut String, state: &mut
}
}
fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem], thumbnails: &mut dyn FnMut(&Ui, &str) -> Option<egui::TextureHandle>) {
fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem], thumbnails: &mut dyn FnMut(&Ui, &str) -> Option<egui::TextureHandle>) -> Option<(String, Option<String>)> {
let selected = state.mention.as_ref().map_or(0, |q| q.selected);
let row_h = 44.0;
let h = 28.0 + row_h * filtered.len() as f32;
@@ -317,23 +327,25 @@ fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &m
MentionKind::Image => Icon::Image,
MentionKind::Video => Icon::Video,
MentionKind::Audio => Icon::Music,
MentionKind::Character => Icon::User,
};
icons::paint(ui.painter(), tile.shrink(9.0), icon, theme::fade(Color32::WHITE, 0.6));
},
}
ui.painter().text(pos2(tile.right() + 10.0, row.center().y), Align2::LEFT_CENTER, &item.label, FontId::new(14.0, theme::medium()), item.color());
if resp.clicked() {
picked = Some(item.label.clone());
picked = Some((item.label.clone(), item.token.clone()));
}
}
});
if let (Some(i), Some(q)) = (hovered, state.mention.as_mut()) {
q.selected = i;
}
if let Some(label) = picked {
insert_mention(ui.ctx(), id, prompt, state, &label);
if let Some((label, _)) = &picked {
insert_mention(ui.ctx(), id, prompt, state, label);
ui.memory_mut(|m| m.request_focus(id));
}
picked
}
/// The three-line grip that resizes the field (`.promptbox-resize-wrap::after`).
@@ -382,14 +394,20 @@ fn highlight(text: &str, mentions: &[MentionItem]) -> LayoutJob {
job
}
/// `@Name` mentions for characters (newest first), coloured by position.
pub fn character_mentions<'a>(characters: impl Iterator<Item = (&'a str, &'a str, Option<String>)>) -> Vec<MentionItem> {
characters.enumerate().map(|(i, (name, token, avatar))| MentionItem { label: format!("@{name}"), kind: MentionKind::Character, index: i + 1, preview: avatar, token: Some(token.to_owned()) }).collect()
}
/// Mention labels for `n` items of `kind` (`@Image1`, `@Image2`, …).
pub fn mention_items(kind: MentionKind, previews: impl Iterator<Item = Option<String>>) -> Vec<MentionItem> {
let name = match kind {
MentionKind::Image => "Image",
MentionKind::Video => "Video",
MentionKind::Audio => "Audio",
MentionKind::Character => "Character",
};
previews.enumerate().map(|(i, preview)| MentionItem { label: format!("@{name}{}", i + 1), kind, index: i + 1, preview }).collect()
previews.enumerate().map(|(i, preview)| MentionItem { label: format!("@{name}{}", i + 1), kind, index: i + 1, preview, token: None }).collect()
}
#[cfg(test)]
+35 -14
View File
@@ -58,6 +58,21 @@ pub struct PromptBoxProps<'a> {
pub warning: Option<&'a str>,
/// An info strip above the prompt.
pub banner: Option<&'a str>,
/// Something clear-all also clears besides the prompt and references (the audio style).
pub extra_input: bool,
}
/// Where a page draws into the box.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Slot {
/// Rows above the prompt (the audio box's references).
AbovePrompt,
/// Rows below the prompt (the audio box's style).
BelowPrompt,
/// The toolbar's left: the model selector and pickers.
ToolbarLeft,
/// The toolbar's right, before clear-all: the count picker.
ToolbarRight,
}
/// Something the page should do.
@@ -68,6 +83,8 @@ pub enum PromptBoxAction {
ClearAll,
DroppedFiles(Vec<PathBuf>),
PastedImage(Vec<u8>),
/// A mention picked from the dropdown: (label, character token).
MentionPicked(String, Option<String>),
}
/// UI state that outlives a frame.
@@ -79,11 +96,10 @@ pub struct PromptBoxState {
focused: bool,
}
/// Draws the box in `ui` (which should be as wide as the box). `toolbar_left` holds the model
/// selector and pickers; `toolbar_right` the count picker. Both may be drawn twice in a frame
/// (box and focus mode), each under its own id.
#[allow(clippy::too_many_arguments)]
pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, toolbar_left: &mut dyn FnMut(&mut Ui), toolbar_right: &mut dyn FnMut(&mut Ui)) -> Vec<PromptBoxAction> {
/// Draws the box in `ui` (which should be as wide as the box). `slots` draws the page's parts
/// into each [`Slot`]; the toolbar's left may be drawn twice in a frame (box and focus mode), each
/// under its own id.
pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, slots: &mut dyn FnMut(&mut Ui, Slot)) -> Vec<PromptBoxAction> {
let mut actions = Vec::new();
let frame = egui::Frame::new().fill(theme::fade(theme::CONTROLS, 0.95)).stroke(Stroke::new(1.0, if state.focused { theme::ACCENT } else { theme::LINE })).corner_radius(theme::RADIUS).inner_margin(16);
let response = frame.show(ui, |ui| {
@@ -94,25 +110,30 @@ pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut Pro
ui.spacing_mut().item_spacing.x = 12.0;
deck_widget(ui, props, refs, cache, false, &mut actions);
ui.vertical(|ui| {
slots(ui, Slot::AbovePrompt);
let out = editor_ui(ui, prompt, state, props, cache, None);
state.focused = out.focused;
if out.submit {
actions.push(PromptBoxAction::Generate);
}
if let Some((label, token)) = out.picked {
actions.push(PromptBoxAction::MentionPicked(label, token));
}
if out.open_fullscreen {
state.fullscreen = true;
}
slots(ui, Slot::BelowPrompt);
});
});
ui.add_space(14.0);
ui.horizontal(|ui| {
ui.spacing_mut().item_spacing.x = 8.0;
toolbar_left(ui);
slots(ui, Slot::ToolbarLeft);
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
ui.spacing_mut().item_spacing.x = 6.0;
generate_cluster(ui, props, &mut actions);
clear_all_button(ui, prompt, refs, state, &mut actions);
toolbar_right(ui);
clear_all_button(ui, prompt, refs, state, props.extra_input, &mut actions);
slots(ui, Slot::ToolbarRight);
if let Some(warning) = props.warning {
warning_label(ui, warning);
}
@@ -124,7 +145,7 @@ pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut Pro
drop_target(ui, props, rect, &mut actions);
paste_images(ui, props, &mut actions);
if state.fullscreen {
focus_mode(ui.ctx(), prompt, refs, state, props, cache, toolbar_left, &mut actions);
focus_mode(ui.ctx(), prompt, refs, state, props, cache, slots, &mut actions);
}
if state.confirm_clear {
confirm_clear(ui.ctx(), props.id, state, &mut actions);
@@ -167,10 +188,10 @@ fn generate_cluster(ui: &mut Ui, props: &PromptBoxProps<'_>, actions: &mut Vec<P
}
/// The eraser button and the hairline after it (`PromptClearAllButton`).
fn clear_all_button(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, actions: &mut Vec<PromptBoxAction>) {
fn clear_all_button(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, extra_input: bool, actions: &mut Vec<PromptBoxAction>) {
let (rule, _) = ui.allocate_exact_size(vec2(1.0, 20.0), Sense::hover());
ui.painter().rect_filled(rule, 0.0, theme::fade(theme::INK, 0.15));
let clearable = !prompt.is_empty() || !refs.is_empty();
let clearable = !prompt.is_empty() || !refs.is_empty() || extra_input;
let resp = ui.add_enabled_ui(clearable, |ui| widgets::ghost(ui, Icon::Eraser, None, "Clear all", false)).inner;
if resp.clicked() {
if refs.is_empty() {
@@ -307,7 +328,7 @@ fn paste_images(ui: &mut Ui, props: &PromptBoxProps<'_>, actions: &mut Vec<Promp
/// Focus mode: a large editor in a modal, with the deck and toolbar at hand.
#[allow(clippy::too_many_arguments)]
fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, toolbar_left: &mut dyn FnMut(&mut Ui), actions: &mut Vec<PromptBoxAction>) {
fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, slots: &mut dyn FnMut(&mut Ui, Slot), actions: &mut Vec<PromptBoxAction>) {
let height = ctx.content_rect().height() * 0.7;
let mut done = false;
let closed = widgets::modal(ctx, props.id.with("focus-mode"), 896.0, |ui| {
@@ -330,12 +351,12 @@ fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state
ui.push_id("focus-deck", |ui| deck_widget(ui, props, refs, cache, true, actions));
ui.add_space(12.0);
ui.horizontal(|ui| {
ui.push_id("focus-toolbar", |ui| ui.horizontal(|ui| toolbar_left(ui)));
ui.push_id("focus-toolbar", |ui| ui.horizontal(|ui| slots(ui, Slot::ToolbarLeft)));
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
if widgets::button(ui, None, "Done", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() {
done = true;
}
ui.push_id("focus-clear", |ui| clear_all_button(ui, prompt, refs, state, actions));
ui.push_id("focus-clear", |ui| clear_all_button(ui, prompt, refs, state, props.extra_input, actions));
});
});
});
+9 -4
View File
@@ -90,14 +90,19 @@ pub fn popover(anchor: &Response, popup_id: Id, content: impl FnOnce(&mut Ui)) {
});
}
/// A toolbar pill that opens a popover of custom `content` above it.
pub fn popover_pill(ui: &mut Ui, id: Id, lead: Leading<'_>, label: &str, tooltip: &str, content: impl FnOnce(&mut Ui)) {
let popup_id = ui.make_persistent_id(id).with("popup");
let open = Popup::is_id_open(ui.ctx(), popup_id);
let resp = ui.push_id(id, |ui| widgets::pill(ui, lead, label, open)).inner.on_hover_text(tooltip);
popover(&resp, popup_id, content);
}
/// The duration popover: a clock pill reading `5s`, opening a 1-second-step slider between the
/// model's minimum and maximum. Returns the new value.
pub fn duration_picker(ui: &mut Ui, id: Id, current: u32, min: u32, max: u32) -> Option<u32> {
let popup_id = ui.make_persistent_id(id).with("popup");
let open = Popup::is_id_open(ui.ctx(), popup_id);
let resp = ui.push_id(id, |ui| widgets::pill(ui, Leading::Icon(Icon::Clock), &format!("{current}s"), open)).inner.on_hover_text("Duration");
let mut value = current;
popover(&resp, popup_id, |ui| {
popover_pill(ui, id, Leading::Icon(Icon::Clock), &format!("{current}s"), "Duration", |ui| {
ui.set_width(240.0);
widgets::menu_header(ui, "Duration");
ui.add_space(4.0);
+9 -8
View File
@@ -11,7 +11,7 @@ use crate::pages::toolbar;
use crate::prompt_box::deck::DeckLimits;
use crate::prompt_box::editor::{MentionKind, mention_items};
use crate::prompt_box::refs::{RefKind, References};
use crate::prompt_box::{DeckMode, PromptBoxProps, PromptBoxState, show};
use crate::prompt_box::{DeckMode, PromptBoxProps, PromptBoxState, Slot, show};
struct State {
// Keeps the media cache's runtime alive.
@@ -45,15 +45,16 @@ fn build(state: State) -> Harness<'static, State> {
return;
}
let mentions = if s.with_mentions { mention_items(MentionKind::Image, [None, None].into_iter()) } else { Vec::new() };
let props = PromptBoxProps { id: egui::Id::new("test-box"), placeholder: "Describe what you want in the image...", enter_to_generate: s.enter_to_generate, mentions: &mentions, max_length: Some(Some(1000)), deck: DeckMode::References(DeckLimits { max_images: 4, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: false }), accepts: &[RefKind::Image], credits: Some(15), generate_enabled: !s.prompt.trim().is_empty(), generating: false, generate_tooltip: "Generate", warning: None, banner: None };
let props = PromptBoxProps { id: egui::Id::new("test-box"), placeholder: "Describe what you want in the image...", enter_to_generate: s.enter_to_generate, mentions: &mentions, max_length: Some(Some(1000)), deck: DeckMode::References(DeckLimits { max_images: 4, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: false }), accepts: &[RefKind::Image], credits: Some(15), generate_enabled: !s.prompt.trim().is_empty(), generating: false, generate_tooltip: "Generate", warning: None, banner: None, extra_input: false };
let (model, aspect) = (&s.model, &mut s.aspect);
let mut left = |ui: &mut egui::Ui| {
if let Some(v) = toolbar::aspect_ratio(ui, egui::Id::new("test-aspect"), model, aspect) {
*aspect = v;
let mut slots = |ui: &mut egui::Ui, slot: Slot| {
if slot == Slot::ToolbarLeft {
if let Some(v) = toolbar::aspect_ratio(ui, egui::Id::new("test-aspect"), model, aspect) {
*aspect = v;
}
}
};
let mut right = |_: &mut egui::Ui| {};
let actions = show(ui, &mut s.prompt, &s.refs, &mut s.box_state, &props, &mut s.cache, &mut left, &mut right);
let actions = show(ui, &mut s.prompt, &s.refs, &mut s.box_state, &props, &mut s.cache, &mut slots);
s.actions.extend(actions.iter().map(|a| format!("{a:?}")));
},
state,
@@ -128,7 +129,7 @@ fn typing_at_offers_mentions_and_enter_inserts_one() {
harness.key_press(egui::Key::Enter);
harness.run();
assert_eq!(harness.state().prompt, "put @Image2 ");
assert!(harness.state().actions.is_empty(), "Enter picked a mention, it didn't generate");
assert_eq!(harness.state().actions, [r#"MentionPicked("@Image2", None)"#], "Enter picked a mention, it didn't generate");
}
#[test]
+96 -70
View File
@@ -1,16 +1,19 @@
//! The app frame from the webapp: a floating sidebar (`app-sidebar`) and the top bar with
//! breadcrumbs, feed toggles, credits and the account menu (`topbar`).
//! The app frame: a custom title bar (mark, sidebar toggle, account, caption buttons), a sidebar
//! flush with the window edge, and the content panel's header (breadcrumbs and feed toggles).
use egui::{Align2, Color32, FontId, Id, Rect, Sense, Stroke, Ui, pos2, vec2};
use crate::feed::grid::ViewMode;
use crate::theme;
use crate::ui::icons::{self, Icon};
use crate::ui::widgets;
use crate::ui::{widgets, window};
const SIDEBAR_WIDTH: f32 = 232.0;
const SIDEBAR_INSET: f32 = 8.0;
const TOPBAR_HEIGHT: f32 = 57.0;
pub const SIDEBAR_WIDTH: f32 = 232.0;
pub const TITLEBAR_HEIGHT: f32 = 40.0;
pub const HEADER_HEIGHT: f32 = 48.0;
/// The content panel's gap to the window edges, and its corner radius (Claude-app style).
pub const PANEL_GAP: i8 = 8;
pub const PANEL_RADIUS: u8 = 8;
const ROW_HEIGHT: f32 = 32.0;
pub const DISCORD_URL: &str = "https://discord.gg/artcraft";
pub const GITHUB_URL: &str = "https://github.com/storytold/artcraft";
@@ -84,6 +87,7 @@ impl Page {
"home" => Page::Home,
"create-image" | "image" => Page::CreateImage,
"create-video" | "video" => Page::CreateVideo,
"create-audio" | "audio" => Page::CreateAudio,
"library" => Page::Library,
_ => return None,
})
@@ -91,7 +95,7 @@ impl Page {
/// Pages the native app has; the rest open a "coming soon" page.
pub fn is_ready(self) -> bool {
matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::Library)
matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::CreateAudio | Page::Library)
}
/// The same tool in the web app.
@@ -130,29 +134,70 @@ pub enum ShellAction {
SignOut,
OpenSettings,
SetViewMode(ViewMode),
SetAutoplay(bool),
ToggleSelect,
}
/// The sidebar. Returns the user's action, if any.
pub fn sidebar(ui: &mut Ui, page: Page, account: &Account<'_>, logo: &egui::TextureHandle) -> Option<ShellAction> {
/// The custom title bar: the ArtCraft mark (home) and the sidebar toggle at the left, the
/// account and (on Windows and Linux) the caption buttons at the right; the rest drags the window.
pub fn titlebar(ui: &mut Ui, account: &Account<'_>, mark: &egui::TextureHandle) -> Option<ShellAction> {
let mut action = None;
let rect = ui.max_rect().shrink(SIDEBAR_INSET);
ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, theme::hairline(), egui::StrokeKind::Inside);
let mut inner = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink2(vec2(14.0, 10.0))));
window::title_bar_drag(ui);
let rect = ui.max_rect();
let mut bar = ui.new_child(egui::UiBuilder::new().max_rect(rect).layout(egui::Layout::left_to_right(egui::Align::Center)));
let ui = &mut bar;
ui.spacing_mut().item_spacing.x = 4.0;
ui.add_space(window::leading_inset() + 10.0);
let (mark_rect, mark_resp) = ui.allocate_exact_size(vec2(28.0, 28.0), Sense::click());
let hovered = mark_resp.hovered();
if hovered {
ui.painter().rect_filled(mark_rect, theme::RADIUS, theme::WASH);
}
ui.painter().image(mark.id(), Rect::from_center_size(mark_rect.center(), vec2(18.0, 16.0)), Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), theme::ACCENT);
if mark_resp.on_hover_text("Home").on_hover_cursor(egui::CursorIcon::PointingHand).clicked() {
action = Some(ShellAction::Navigate(Page::Home));
}
if widgets::ghost(ui, Icon::PanelLeft, None, "Toggle Sidebar", false).clicked() {
action = Some(ShellAction::ToggleSidebar);
}
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
ui.spacing_mut().item_spacing.x = 8.0;
if window::custom_caption_buttons() {
ui.spacing_mut().item_spacing.x = 0.0;
window::caption_buttons(ui, TITLEBAR_HEIGHT);
ui.add_space(12.0);
ui.spacing_mut().item_spacing.x = 8.0;
} else {
ui.add_space(12.0);
}
match account.username {
Some(_) => {
if let Some(a) = account_menu(ui, account) {
action = Some(a);
}
if let Some(credits) = account.credits {
if credits_pill(ui, credits).clicked() {
ui.ctx().open_url(egui::OpenUrl::new_tab(format!("{WEB_APP_URL}/pricing")));
}
}
},
None => {
if widgets::button(ui, None, "Sign in", widgets::Kind::Primary, 0.0, 28.0).clicked() {
action = Some(ShellAction::SignIn);
}
},
}
});
action
}
/// The sidebar, flush with the window's left edge. Returns the user's action, if any.
pub fn sidebar(ui: &mut Ui, page: Page, account: &Account<'_>) -> Option<ShellAction> {
let mut action = None;
let rect = ui.max_rect();
let mut inner = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink2(vec2(10.0, 4.0))));
let ui = &mut inner;
ui.spacing_mut().item_spacing.y = 2.0;
ui.horizontal(|ui| {
let h = 20.0;
let w = h * logo.size_vec2().x / logo.size_vec2().y.max(1.0);
ui.add(egui::Image::new(logo).fit_to_exact_size(vec2(w, h)));
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
if widgets::ghost(ui, Icon::PanelLeft, None, "Toggle Sidebar", false).clicked() {
action = Some(ShellAction::ToggleSidebar);
}
});
});
ui.add_space(6.0);
if nav_row(ui, Page::Home, page == Page::Home) {
action = Some(ShellAction::Navigate(Page::Home));
}
@@ -174,12 +219,11 @@ pub fn sidebar(ui: &mut Ui, page: Page, account: &Account<'_>, logo: &egui::Text
}
// The account sits at the bottom, like the webapp's download button.
let bottom = Rect::from_min_max(pos2(rect.left() + 10.0, rect.bottom() - 58.0), pos2(rect.right() - 10.0, rect.bottom() - 10.0));
let bottom = Rect::from_min_max(pos2(rect.left() + 10.0, rect.bottom() - 56.0), pos2(rect.right() - 10.0, rect.bottom() - 8.0));
let mut foot = ui.new_child(egui::UiBuilder::new().max_rect(bottom));
match account.username {
Some(username) => {
let resp = account_row(&mut foot, account.display_name.unwrap_or(username), account.credits);
if resp.clicked() {
if account_row(&mut foot, account.display_name.unwrap_or(username), account.credits).clicked() {
action = Some(ShellAction::OpenSettings);
}
},
@@ -192,11 +236,6 @@ pub fn sidebar(ui: &mut Ui, page: Page, account: &Account<'_>, logo: &egui::Text
action
}
/// The sidebar's full width (including its inset), for the panel.
pub fn sidebar_width() -> f32 {
SIDEBAR_WIDTH + SIDEBAR_INSET * 2.0
}
fn nav_row(ui: &mut Ui, page: Page, selected: bool) -> bool {
let (rect, resp) = ui.allocate_exact_size(vec2(ui.available_width(), ROW_HEIGHT), Sense::click());
let hovered = resp.hovered();
@@ -249,63 +288,50 @@ fn account_row(ui: &mut Ui, name: &str, credits: Option<u64>) -> egui::Response
resp.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_text("Settings")
}
/// The top bar. `feed_toggles` shows the select and grid/list toggles (create pages).
pub fn topbar(ui: &mut Ui, page: Page, account: &Account<'_>, feed_toggles: Option<(ViewMode, bool)>, sidebar_open: bool) -> Option<ShellAction> {
/// The feed toggles a page's header shows.
pub struct FeedToggles {
pub mode: ViewMode,
pub selecting: bool,
/// Shown on pages with videos (`GalleryAutoplayToggle`).
pub autoplay: Option<bool>,
}
/// The content panel's header: breadcrumbs, and the select, previews and grid/list toggles on
/// feed pages.
pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option<FeedToggles>) -> Option<ShellAction> {
let mut action = None;
let rect = ui.max_rect();
ui.painter().rect_filled(rect, 0.0, theme::PANEL);
ui.painter().hline(rect.x_range(), rect.bottom() - 0.5, theme::hairline());
let mut bar = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink2(vec2(20.0, 0.0))).layout(egui::Layout::left_to_right(egui::Align::Center)));
let ui = &mut bar;
ui.spacing_mut().item_spacing.x = 8.0;
if !sidebar_open && widgets::ghost(ui, Icon::PanelLeft, None, "Toggle Sidebar", false).clicked() {
action = Some(ShellAction::ToggleSidebar);
}
let (section, leaf) = page.crumbs();
widgets::hud(ui, section, theme::MUTED);
let (chev, _) = ui.allocate_exact_size(vec2(10.0, 10.0), Sense::hover());
icons::paint(ui.painter(), chev, Icon::ChevronRight, theme::FAINT);
let galley = widgets::caps(ui, leaf, theme::HUD_SIZE, theme::mono(), theme::HUD_SPACING, theme::PANEL);
let galley = widgets::caps(ui, leaf, theme::HUD_SIZE, theme::mono(), theme::HUD_SPACING, theme::BG);
let (crumb, _) = ui.allocate_exact_size(galley.size() + vec2(12.0, 8.0), Sense::hover());
ui.painter().rect_filled(crumb, 0.0, theme::INK);
ui.painter().galley(crumb.center() - galley.size() / 2.0, galley, theme::PANEL);
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
ui.spacing_mut().item_spacing.x = 8.0;
match account.username {
Some(_) => {
if let Some(a) = account_menu(ui, account) {
action = Some(a);
}
if let Some(credits) = account.credits {
let resp = credits_pill(ui, credits);
if resp.clicked() {
ui.ctx().open_url(egui::OpenUrl::new_tab(format!("{WEB_APP_URL}/pricing")));
}
}
},
None => {
if widgets::button(ui, None, "Sign in", widgets::Kind::Primary, 0.0, 32.0).clicked() {
action = Some(ShellAction::SignIn);
}
},
}
if let Some((mode, selecting)) = feed_toggles {
if let Some(m) = view_toggle(ui, mode) {
ui.painter().galley(crumb.center() - galley.size() / 2.0, galley, theme::BG);
if let Some(t) = feed_toggles {
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
if let Some(m) = view_toggle(ui, t.mode) {
action = Some(ShellAction::SetViewMode(m));
}
if segmented(ui, &[(Icon::SquareCheck, if selecting { "Exit selection" } else { "Select items" }, selecting)]).is_some() {
if let Some(playing) = t.autoplay {
// Right-to-left: stills first so "play" ends up on the left.
if let Some(i) = segmented(ui, &[(Icon::Image, "Still thumbnails", !playing), (Icon::Play, "Play video previews", playing)]) {
action = Some(ShellAction::SetAutoplay(i == 1));
}
}
if segmented(ui, &[(Icon::SquareCheck, if t.selecting { "Exit selection" } else { "Select items" }, t.selecting)]).is_some() {
action = Some(ShellAction::ToggleSelect);
}
}
});
});
}
action
}
pub fn topbar_height() -> f32 {
TOPBAR_HEIGHT
}
fn credits_pill(ui: &mut Ui, credits: u64) -> egui::Response {
let text = group_thousands(credits);
let galley = ui.painter().layout_no_wrap(text, FontId::new(13.0, theme::semibold()), theme::INK);
+2
View File
@@ -24,6 +24,8 @@ pub const BG: Color32 = hex(0x121316);
pub const PANEL: Color32 = hex(0x101014);
/// `ui-controls`: the sidebar, the prompt box and toolbar buttons.
pub const CONTROLS: Color32 = hex(0x1e1f22);
/// The window chrome behind the content panel: the title bar and the sidebar.
pub const CHROME: Color32 = hex(0x0a0a0c);
/// `ui-sunken`: wells and inputs.
pub const SUNKEN: Color32 = hex(0x060607);
+54
View File
@@ -18,6 +18,7 @@ pub enum Icon {
Coins,
Copy,
Download,
Drum,
Eraser,
Eye,
ExternalLink,
@@ -35,17 +36,24 @@ pub enum Icon {
LogIn,
LogOut,
Maximize,
Mic,
MicOff,
Music,
PanelLeft,
Pause,
Pencil,
Play,
Plus,
Repeat,
RotateCw,
Settings,
SlidersHorizontal,
Sparkles,
SquareCheck,
Tags,
Trash,
Upload,
User,
Video,
Volume,
VolumeOff,
@@ -157,6 +165,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
g.line(p, &[(7.0, 10.0), (12.0, 15.0), (17.0, 10.0)]);
g.line(p, &[(4.0, 21.0), (20.0, 21.0)]);
},
Icon::Drum => {
g.line(p, &[(2.0, 2.0), (10.0, 8.0)]);
g.line(p, &[(22.0, 2.0), (14.0, 8.0)]);
g.closed(p, &[(2.0, 9.0), (6.0, 6.5), (12.0, 5.5), (18.0, 6.5), (22.0, 9.0), (18.0, 11.5), (12.0, 12.5), (6.0, 11.5)]);
g.line(p, &[(2.0, 9.0), (2.0, 17.0), (6.0, 19.5), (12.0, 20.5), (18.0, 19.5), (22.0, 17.0), (22.0, 9.0)]);
g.line(p, &[(7.0, 11.8), (7.0, 19.8)]);
g.line(p, &[(17.0, 11.8), (17.0, 19.8)]);
},
Icon::Eraser => {
g.closed(p, &[(7.0, 21.0), (3.5, 17.5), (14.0, 7.0), (21.0, 14.0), (14.0, 21.0)]);
g.line(p, &[(22.0, 21.0), (7.0, 21.0)]);
@@ -248,6 +264,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
g.line(p, &[(21.0, 3.0), (14.0, 10.0)]);
g.line(p, &[(3.0, 21.0), (10.0, 14.0)]);
},
Icon::Mic | Icon::MicOff => {
g.rect(p, 9.0, 2.0, 6.0, 13.0, 3.0);
g.line(p, &[(19.0, 10.0), (19.0, 12.0), (17.0, 16.0), (12.0, 19.0), (7.0, 16.0), (5.0, 12.0), (5.0, 10.0)]);
g.line(p, &[(12.0, 19.0), (12.0, 22.0)]);
if icon == Icon::MicOff {
g.line(p, &[(2.0, 2.0), (22.0, 22.0)]);
}
},
Icon::Music => {
g.line(p, &[(9.0, 18.0), (9.0, 5.0), (21.0, 3.0), (21.0, 16.0)]);
g.circle(p, 6.0, 18.0, 3.0);
@@ -257,6 +281,10 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
g.rect(p, 3.0, 3.0, 18.0, 18.0, 2.0);
g.line(p, &[(9.0, 3.0), (9.0, 21.0)]);
},
Icon::Pause => {
p.rect_filled(Rect::from_min_size(g.p(6.0, 4.0), vec2(4.0, 16.0) * g.scale), 1.0, color);
p.rect_filled(Rect::from_min_size(g.p(14.0, 4.0), vec2(4.0, 16.0) * g.scale), 1.0, color);
},
Icon::Pencil => {
g.closed(p, &[(17.0, 3.0), (21.0, 7.0), (7.5, 20.5), (2.0, 22.0), (3.5, 16.5)]);
g.line(p, &[(15.0, 5.0), (19.0, 9.0)]);
@@ -269,6 +297,12 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
g.line(p, &[(5.0, 12.0), (19.0, 12.0)]);
g.line(p, &[(12.0, 5.0), (12.0, 19.0)]);
},
Icon::Repeat => {
g.line(p, &[(17.0, 2.0), (21.0, 6.0), (17.0, 10.0)]);
g.line(p, &[(3.0, 11.0), (3.0, 10.0), (5.0, 7.0), (7.0, 6.0), (21.0, 6.0)]);
g.line(p, &[(7.0, 22.0), (3.0, 18.0), (7.0, 14.0)]);
g.line(p, &[(21.0, 13.0), (21.0, 14.0), (19.0, 17.0), (17.0, 18.0), (3.0, 18.0)]);
},
Icon::RotateCw => {
g.arc(p, 12.0, 12.0, 9.0, -60.0, 250.0);
g.line(p, &[(21.0, 3.0), (21.0, 8.5), (15.5, 8.5)]);
@@ -281,6 +315,17 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
g.line(p, &[(12.0 + 7.5 * a.cos(), 12.0 + 7.5 * a.sin()), (12.0 + 10.0 * a.cos(), 12.0 + 10.0 * a.sin())]);
}
},
Icon::SlidersHorizontal => {
g.line(p, &[(21.0, 4.0), (14.0, 4.0)]);
g.line(p, &[(10.0, 4.0), (3.0, 4.0)]);
g.line(p, &[(21.0, 12.0), (12.0, 12.0)]);
g.line(p, &[(8.0, 12.0), (3.0, 12.0)]);
g.line(p, &[(21.0, 20.0), (16.0, 20.0)]);
g.line(p, &[(12.0, 20.0), (3.0, 20.0)]);
g.line(p, &[(14.0, 2.0), (14.0, 6.0)]);
g.line(p, &[(8.0, 10.0), (8.0, 14.0)]);
g.line(p, &[(16.0, 18.0), (16.0, 22.0)]);
},
Icon::Sparkles => {
let star = |cx: f32, cy: f32, r: f32| {
let k = r * 0.28;
@@ -289,6 +334,11 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
star(10.0, 12.0, 8.0);
star(19.0, 5.0, 3.0);
},
Icon::Tags => {
g.closed(p, &[(3.0, 5.0), (5.0, 3.0), (10.6, 3.0), (18.0, 10.4), (18.0, 12.6), (12.6, 18.0), (10.4, 18.0), (3.0, 10.6)]);
g.line(p, &[(15.0, 3.0), (22.0, 10.0), (22.0, 12.0), (17.0, 17.0)]);
g.dot(p, 7.5, 7.5, 1.1);
},
Icon::Trash => {
g.line(p, &[(3.0, 6.0), (21.0, 6.0)]);
g.line(p, &[(19.0, 6.0), (19.0, 20.0), (17.0, 22.0), (7.0, 22.0), (5.0, 20.0), (5.0, 6.0)]);
@@ -299,6 +349,10 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
g.line(p, &[(7.0, 8.0), (12.0, 3.0), (17.0, 8.0)]);
g.line(p, &[(4.0, 21.0), (20.0, 21.0)]);
},
Icon::User => {
g.circle(p, 12.0, 8.0, 4.5);
g.arc(p, 12.0, 22.0, 8.0, 180.0, 360.0);
},
Icon::Video => {
g.rect(p, 2.0, 6.0, 14.0, 12.0, 2.0);
g.closed(p, &[(16.0, 10.5), (22.0, 7.0), (22.0, 17.0), (16.0, 13.5)]);
+2
View File
@@ -1,4 +1,6 @@
pub mod creator_icons;
pub mod icons;
pub mod toast;
pub mod waveform;
pub mod widgets;
pub mod window;
+2 -2
View File
@@ -9,8 +9,8 @@ use crate::ui::icons::{self, Icon};
const DEFAULT_DURATION: Duration = Duration::from_secs(4);
const MAX_WIDTH: f32 = 448.0;
/// Below the top bar, like the webapp's `top-16 right-5`.
const TOP_OFFSET: f32 = 64.0;
/// Below the title bar and the panel header (the webapp's `top-16 right-5`, shifted).
const TOP_OFFSET: f32 = 100.0;
const RIGHT_OFFSET: f32 = 20.0;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+194
View File
@@ -0,0 +1,194 @@
//! The track player (`WaveformAudioPlayer`): a round white play button, the waveform (click to
//! seek, played part in blue), the time, and a volume flyout. Placeholder bars stand in while the
//! waveform decodes; a plain progress line if it can't.
use egui::{Color32, FontId, Id, Popup, Rect, Sense, Stroke, Ui, pos2, vec2};
use crate::backend::audio::{self, AudioPlayer, WaveformLookup};
use crate::prompt_box::pickers;
use crate::theme;
use crate::ui::icons::{self, Icon};
/// Bars drawn while the waveform loads (`PLACEHOLDER_BAR_COUNT`).
const PLACEHOLDER_BARS: usize = 48;
/// Waveform bars: width and the step between them.
const BAR_WIDTH: f32 = 2.0;
const BAR_STEP: f32 = 3.5;
const WAVE_INK: Color32 = Color32::from_rgba_premultiplied(89, 89, 89, 89);
const PROGRESS_INK: Color32 = Color32::from_rgb(0x2d, 0x81, 0xff);
/// How often a playing track redraws its progress.
const FRAME: std::time::Duration = std::time::Duration::from_millis(33);
/// Draws the player across `ui`'s width; `duration_hint` (seconds) is used until the waveform
/// knows the real length. `compact` is the list rows' smaller size. Returns the player's rect.
pub fn player(ui: &mut Ui, id: Id, audio: &mut AudioPlayer, url: &str, duration_hint: Option<f32>, compact: bool) -> Rect {
let (button_d, wave_h): (f32, f32) = if compact { (32.0, 32.0) } else { (40.0, 44.0) };
let (rect, _) = ui.allocate_exact_size(vec2(ui.available_width(), wave_h.max(button_d)), Sense::hover());
// Clicks between the controls stay with the player (they don't open the card underneath).
ui.interact(rect, id.with("bg"), Sense::click());
let ctx = ui.ctx().clone();
let waveform = audio.waveform(&ctx, url);
let playback = audio.track(url);
let duration = match &waveform {
WaveformLookup::Ready(w) => Some(w.duration),
_ => duration_hint,
}
.filter(|d| *d > 0.0);
let position = playback.map_or(0.0, |p| p.position);
let playing = playback.is_some_and(|p| !p.paused && !p.loading);
if playing {
ctx.request_repaint_after(FRAME);
}
// Play / pause.
let button = Rect::from_center_size(pos2(rect.left() + button_d / 2.0, rect.center().y), vec2(button_d, button_d));
let resp = ui.interact(button, id.with("play"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
let label = if playing { "Pause" } else { "Play" };
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
ui.painter().circle_filled(button.center(), button_d / 2.0, if resp.hovered() { theme::fade(Color32::WHITE, 0.88) } else { Color32::WHITE });
let glyph = Rect::from_center_size(button.center(), vec2(button_d * 0.4, button_d * 0.4));
match playback {
Some(p) if p.loading => icons::paint_spinner(ui, glyph, Color32::BLACK),
_ if playing => icons::paint(ui.painter(), glyph, Icon::Pause, Color32::BLACK),
_ => icons::paint(ui.painter(), glyph.translate(vec2(button_d * 0.04, 0.0)), Icon::Play, Color32::BLACK),
}
if resp.clicked() {
audio.toggle_track(&ctx, url);
}
// Volume, then the time, from the right.
let volume_rect = Rect::from_center_size(pos2(rect.right() - 14.0, rect.center().y), vec2(28.0, 28.0));
volume(ui, id, volume_rect, audio);
let time = format!("{} / {}", format_time(position), duration.map_or_else(|| "-:--".to_owned(), format_time));
let galley = ui.painter().layout_no_wrap(time, FontId::new(11.0, theme::mono()), theme::MUTED);
let time_x = volume_rect.left() - 8.0 - galley.size().x;
ui.painter().galley(pos2(time_x, rect.center().y - galley.size().y / 2.0), galley, theme::MUTED);
// The waveform between them.
let wave = Rect::from_min_max(pos2(button.right() + 10.0, rect.center().y - wave_h / 2.0), pos2(time_x - 10.0, rect.center().y + wave_h / 2.0));
if wave.width() > 8.0 {
let progress = duration.map_or(0.0, |d| (position / d).clamp(0.0, 1.0));
let wave_resp = ui.interact(wave, id.with("wave"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
match &waveform {
WaveformLookup::Ready(w) => paint_bars(ui, wave, &audio::resample(&w.peaks, bar_count(wave.width())), progress),
WaveformLookup::Loading => paint_bars(ui, wave, &placeholder_bars(url), progress),
WaveformLookup::Failed => paint_line(ui, wave, progress),
}
if let Some(pointer) = wave_resp.interact_pointer_pos().filter(|_| wave_resp.clicked()) {
let frac = ((pointer.x - wave.left()) / wave.width()).clamp(0.0, 1.0);
audio.seek_track(&ctx, url, duration.map_or(0.0, |d| frac * d));
}
}
rect
}
/// "m:ss".
pub fn format_time(secs: f32) -> String {
let s = secs.max(0.0) as u64;
format!("{}:{:02}", s / 60, s % 60)
}
/// The volume button and its 0–100 slider flyout.
fn volume(ui: &mut Ui, id: Id, rect: Rect, audio: &mut AudioPlayer) {
let popup_id = id.with("volume-popup");
let open = Popup::is_id_open(ui.ctx(), popup_id);
let resp = ui.interact(rect, id.with("volume"), Sense::click()).on_hover_text("Volume").on_hover_cursor(egui::CursorIcon::PointingHand);
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, "Volume"));
if resp.hovered() || open {
ui.painter().rect_filled(rect, theme::RADIUS, theme::WASH);
}
let level = audio.track_volume();
icons::paint(ui.painter(), rect.shrink(7.0), if level <= 0.0 { Icon::VolumeOff } else { Icon::Volume }, if resp.hovered() || open { theme::INK } else { theme::DIM });
let mut percent = (level * 100.0).round() as u32;
pickers::popover(&resp, popup_id, |ui| {
ui.horizontal(|ui| {
ui.add_space(6.0);
ui.style_mut().spacing.slider_width = 120.0;
ui.add(egui::Slider::new(&mut percent, 0..=100).show_value(false));
ui.label(egui::RichText::new(percent.to_string()).size(11.0).family(theme::mono()).color(theme::MUTED));
ui.add_space(4.0);
});
});
if percent != (level * 100.0).round() as u32 {
audio.set_track_volume(percent as f32 / 100.0);
}
}
fn bar_count(width: f32) -> usize {
((width + BAR_STEP - BAR_WIDTH) / BAR_STEP).floor().max(1.0) as usize
}
/// Bars centred on the waveform's midline, blue up to `progress` (0–1).
fn paint_bars(ui: &Ui, rect: Rect, heights: &[f32], progress: f32) {
if heights.is_empty() {
return;
}
let step = rect.width() / heights.len() as f32;
let width = BAR_WIDTH.min(step * 0.7);
let split = rect.left() + rect.width() * progress;
let p = ui.painter();
for (i, h) in heights.iter().enumerate() {
let x = rect.left() + step * (i as f32 + 0.5);
let half = (h.clamp(0.0, 1.0) * rect.height() / 2.0).max(1.0);
let bar = Rect::from_center_size(pos2(x, rect.center().y), vec2(width, half * 2.0));
p.rect_filled(bar, 1.0, if x <= split { PROGRESS_INK } else { WAVE_INK });
}
}
/// The fallback when the waveform can't be decoded: a thin track with the played part in blue.
fn paint_line(ui: &Ui, rect: Rect, progress: f32) {
let track = Rect::from_center_size(rect.center(), vec2(rect.width(), 4.0));
ui.painter().rect_filled(track, 2.0, WAVE_INK);
ui.painter().rect_filled(Rect::from_min_size(track.min, vec2(track.width() * progress, track.height())), 2.0, PROGRESS_INK);
ui.painter().line_segment([pos2(track.left() + track.width() * progress, rect.top() + 6.0), pos2(track.left() + track.width() * progress, rect.bottom() - 6.0)], Stroke::new(1.0, PROGRESS_INK));
}
/// Deterministic stand-in bars for a track (seeded by its URL), 25–89 % tall.
pub fn placeholder_bars(url: &str) -> Vec<f32> {
let mut seed = url.chars().fold(0_u32, |s, c| s.wrapping_mul(31).wrapping_add(c as u32));
(0..PLACEHOLDER_BARS)
.map(|_| {
let percent = 25 + seed % 65;
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
percent as f32 / 100.0
})
.collect()
}
/// A tile with a music note (`MusicIcon` in its box), for audio without artwork.
pub fn music_tile(ui: &Ui, rect: Rect, round: bool) {
let p = ui.painter();
if round {
p.circle_filled(rect.center(), rect.width() / 2.0, theme::WASH_HOVER);
} else {
p.rect(rect, theme::RADIUS, theme::WASH, theme::hairline(), egui::StrokeKind::Inside);
}
icons::paint(p, Rect::from_center_size(rect.center(), rect.size() * 0.42), Icon::Music, theme::fade(Color32::WHITE, 0.75));
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn times_read_minutes_and_seconds() {
assert_eq!(format_time(0.0), "0:00");
assert_eq!(format_time(65.9), "1:05");
assert_eq!(format_time(600.0), "10:00");
}
#[test]
fn placeholder_bars_are_stable_per_track() {
let a = placeholder_bars("https://cdn/a.mp3");
assert_eq!(a.len(), PLACEHOLDER_BARS);
assert_eq!(a, placeholder_bars("https://cdn/a.mp3"));
assert_ne!(a, placeholder_bars("https://cdn/b.mp3"));
assert!(a.iter().all(|h| (0.25..0.9).contains(h)));
}
#[test]
fn bars_fill_the_width() {
assert_eq!(bar_count(2.0), 1);
assert_eq!(bar_count(352.0), 101);
}
}
+155
View File
@@ -0,0 +1,155 @@
//! Window chrome for the undecorated window: dragging and maximizing from the title bar, the
//! Windows-style caption buttons, and resizing from the edges.
use egui::{Color32, Rect, Response, Sense, Stroke, Ui, vec2};
use crate::theme;
/// How far in from the window's edges a drag resizes. Title bar controls leave this band alone.
const RESIZE_EDGE: f32 = 5.0;
/// Caption buttons (Windows' own are 46 × the title bar height).
const BUTTON_WIDTH: f32 = 46.0;
/// The close button's hover red (Windows').
const CLOSE_RED: Color32 = Color32::from_rgb(0xc4, 0x2b, 0x1c);
/// Whether this platform draws its own title bar buttons (macOS keeps the traffic lights).
pub fn custom_caption_buttons() -> bool {
!cfg!(target_os = "macos")
}
/// Space to leave at the title bar's left for macOS's traffic lights.
pub fn leading_inset() -> f32 {
if cfg!(target_os = "macos") {
76.0
} else {
0.0
}
}
/// Dragging the title bar's empty space moves the window; double-clicking maximizes or restores.
/// Call before drawing the title bar's controls so they stay on top.
pub fn title_bar_drag(ui: &mut Ui) {
let r = ui.max_rect();
let rect = Rect::from_min_max(r.min + vec2(RESIZE_EDGE, RESIZE_EDGE), r.max - vec2(RESIZE_EDGE, 0.0));
let resp = ui.interact(rect, ui.id().with("title-bar-drag"), Sense::click_and_drag());
if resp.drag_started_by(egui::PointerButton::Primary) {
ui.ctx().send_viewport_cmd(egui::ViewportCommand::StartDrag);
}
if resp.double_clicked() {
let maximized = ui.input(|i| i.viewport().maximized.unwrap_or(false));
ui.ctx().send_viewport_cmd(egui::ViewportCommand::Maximized(!maximized));
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Caption {
Minimize,
Maximize,
Restore,
Close,
}
/// Minimize, maximize (or restore) and close, `height` tall, for a right-to-left layout so close
/// ends up at the right edge.
pub fn caption_buttons(ui: &mut Ui, height: f32) {
let ctx = ui.ctx().clone();
let maximized = ctx.input(|i| i.viewport().maximized.unwrap_or(false));
if caption_button(ui, Caption::Close, height).clicked() {
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
}
let size = if maximized { Caption::Restore } else { Caption::Maximize };
if caption_button(ui, size, height).clicked() {
ctx.send_viewport_cmd(egui::ViewportCommand::Maximized(!maximized));
}
if caption_button(ui, Caption::Minimize, height).clicked() {
ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true));
}
}
fn caption_button(ui: &mut Ui, button: Caption, height: f32) -> Response {
let (rect, _) = ui.allocate_exact_size(vec2(BUTTON_WIDTH, height), Sense::hover());
// Clickable below the resize band along the top (and, for close, the right edge).
let right = if button == Caption::Close { RESIZE_EDGE } else { 0.0 };
let hit = Rect::from_min_max(rect.min + vec2(0.0, RESIZE_EDGE), rect.max - vec2(right, 0.0));
let resp = ui.interact(hit, ui.id().with(("caption", button as u8)), Sense::click());
let (fill, ink) = match (button, resp.hovered()) {
(Caption::Close, true) => (CLOSE_RED, Color32::WHITE),
(_, true) => (theme::WASH_HOVER, theme::INK_STRONG),
(_, false) => (Color32::TRANSPARENT, theme::MUTED),
};
let p = ui.painter();
p.rect_filled(rect, 0.0, fill);
let c = rect.center();
let stroke = Stroke::new(1.0, ink);
// Half the glyph's size: 10 px, like Windows' own caption buttons.
let s = 5.0;
let label = match button {
Caption::Minimize => {
p.hline(c.x - s..=c.x + s, c.y, stroke);
"Minimize"
},
Caption::Maximize => {
p.rect_stroke(Rect::from_center_size(c, vec2(2.0 * s, 2.0 * s)), 0.0, stroke, egui::StrokeKind::Inside);
"Maximize"
},
Caption::Restore => {
// A window in front of another.
let front = Rect::from_min_size(c + vec2(-s, -s + 2.0), vec2(2.0 * s - 2.0, 2.0 * s - 2.0));
p.rect_stroke(front, 0.0, stroke, egui::StrokeKind::Inside);
p.hline(c.x - s + 2.0..=c.x + s, c.y - s, stroke);
p.vline(c.x + s, c.y - s..=c.y + s - 2.0, stroke);
"Restore"
},
Caption::Close => {
p.line_segment([c + vec2(-s, -s), c + vec2(s, s)], stroke);
p.line_segment([c + vec2(-s, s), c + vec2(s, -s)], stroke);
"Close"
},
};
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
resp.on_hover_text(label)
}
/// A hairline round the window, and resizing from its edges and corners (when not maximized).
pub fn window_edges(ctx: &egui::Context) {
use egui::CursorIcon as C;
use egui::viewport::ResizeDirection as D;
let (maximized, fullscreen) = ctx.input(|i| (i.viewport().maximized.unwrap_or(false), i.viewport().fullscreen.unwrap_or(false)));
if maximized || fullscreen || !custom_caption_buttons() {
return;
}
let r = ctx.content_rect();
ctx.layer_painter(egui::LayerId::new(egui::Order::Foreground, egui::Id::new("window-border"))).rect_stroke(r, 0.0, theme::hairline(), egui::StrokeKind::Inside);
let Some(p) = ctx.input(|i| i.pointer.hover_pos()).filter(|p| r.contains(*p)) else {
return;
};
let near = |d: f32| d < RESIZE_EDGE;
// A corner takes a little of each edge next to it.
let corner = |d: f32| d < 3.0 * RESIZE_EDGE;
let (left, right, top, bottom) = (p.x - r.left(), r.right() - p.x, p.y - r.top(), r.bottom() - p.y);
let hit = if (near(top) && corner(left)) || (near(left) && corner(top)) {
Some((D::NorthWest, C::ResizeNorthWest))
} else if (near(top) && corner(right)) || (near(right) && corner(top)) {
Some((D::NorthEast, C::ResizeNorthEast))
} else if (near(bottom) && corner(left)) || (near(left) && corner(bottom)) {
Some((D::SouthWest, C::ResizeSouthWest))
} else if (near(bottom) && corner(right)) || (near(right) && corner(bottom)) {
Some((D::SouthEast, C::ResizeSouthEast))
} else if near(top) {
Some((D::North, C::ResizeNorth))
} else if near(bottom) {
Some((D::South, C::ResizeSouth))
} else if near(left) {
Some((D::West, C::ResizeWest))
} else if near(right) {
Some((D::East, C::ResizeEast))
} else {
None
};
if let Some((direction, cursor)) = hit {
ctx.set_cursor_icon(cursor);
if ctx.input(|i| i.pointer.primary_pressed()) {
ctx.send_viewport_cmd(egui::ViewportCommand::BeginResize(direction));
}
}
}