mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-20 17:02:53 -04:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
28a39a0de4 |
Generated
+324
-7
@@ -184,6 +184,28 @@ version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
|
||||
|
||||
[[package]]
|
||||
name = "alsa"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43"
|
||||
dependencies = [
|
||||
"alsa-sys",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alsa-sys"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android-activity"
|
||||
version = "0.6.1"
|
||||
@@ -196,9 +218,9 @@ dependencies = [
|
||||
"jni 0.22.4",
|
||||
"libc",
|
||||
"log",
|
||||
"ndk",
|
||||
"ndk 0.9.0",
|
||||
"ndk-context",
|
||||
"ndk-sys",
|
||||
"ndk-sys 0.6.0+11769913",
|
||||
"num_enum",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
@@ -480,6 +502,7 @@ dependencies = [
|
||||
"reqwest 0.12.23",
|
||||
"resvg",
|
||||
"rfd",
|
||||
"rodio",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokens",
|
||||
@@ -1730,6 +1753,49 @@ dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"core-foundation-sys",
|
||||
"coreaudio-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-sys"
|
||||
version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpal"
|
||||
version = "0.15.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779"
|
||||
dependencies = [
|
||||
"alsa",
|
||||
"core-foundation-sys",
|
||||
"coreaudio-rs",
|
||||
"dasp_sample",
|
||||
"jni 0.21.1",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"mach2",
|
||||
"ndk 0.8.0",
|
||||
"ndk-context",
|
||||
"oboe",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"windows 0.54.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.12"
|
||||
@@ -1930,6 +1996,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dasp_sample"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.5.0"
|
||||
@@ -2696,6 +2768,12 @@ dependencies = [
|
||||
"zune-inflate",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "extended"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
|
||||
|
||||
[[package]]
|
||||
name = "fal_client"
|
||||
version = "0.0.1"
|
||||
@@ -4832,6 +4910,15 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.35.0"
|
||||
@@ -5075,6 +5162,20 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"jni-sys 0.3.1",
|
||||
"log",
|
||||
"ndk-sys 0.5.0+25.2.9519653",
|
||||
"num_enum",
|
||||
"thiserror 1.0.63",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.9.0"
|
||||
@@ -5084,7 +5185,7 @@ dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"jni-sys 0.3.1",
|
||||
"log",
|
||||
"ndk-sys",
|
||||
"ndk-sys 0.6.0+11769913",
|
||||
"num_enum",
|
||||
"raw-window-handle",
|
||||
"thiserror 1.0.63",
|
||||
@@ -5096,6 +5197,15 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
|
||||
|
||||
[[package]]
|
||||
name = "ndk-sys"
|
||||
version = "0.5.0+25.2.9519653"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691"
|
||||
dependencies = [
|
||||
"jni-sys 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk-sys"
|
||||
version = "0.6.0+11769913"
|
||||
@@ -5642,6 +5752,29 @@ dependencies = [
|
||||
"objc2-foundation 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oboe"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"
|
||||
dependencies = [
|
||||
"jni 0.21.1",
|
||||
"ndk 0.8.0",
|
||||
"ndk-context",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"oboe-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oboe-sys"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
@@ -6917,6 +7050,16 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rodio"
|
||||
version = "0.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7ceb6607dd738c99bc8cb28eff249b7cd5c8ec88b9db96c0608c1480d140fb1"
|
||||
dependencies = [
|
||||
"cpal",
|
||||
"symphonia",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ron"
|
||||
version = "0.12.2"
|
||||
@@ -8258,6 +8401,151 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"symphonia-bundle-flac",
|
||||
"symphonia-bundle-mp3",
|
||||
"symphonia-codec-aac",
|
||||
"symphonia-codec-adpcm",
|
||||
"symphonia-codec-pcm",
|
||||
"symphonia-codec-vorbis",
|
||||
"symphonia-core",
|
||||
"symphonia-format-isomp4",
|
||||
"symphonia-format-riff",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-bundle-flac"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
"symphonia-utils-xiph",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-bundle-mp3"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-codec-aac"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-codec-adpcm"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-codec-pcm"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-codec-vorbis"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-utils-xiph",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-core"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 1.3.2",
|
||||
"bytemuck",
|
||||
"lazy_static",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-isomp4"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
"symphonia-utils-xiph",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-riff"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f"
|
||||
dependencies = [
|
||||
"extended",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-metadata"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-utils-xiph"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16"
|
||||
dependencies = [
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
@@ -8366,8 +8654,8 @@ dependencies = [
|
||||
"jni 0.21.1",
|
||||
"libc",
|
||||
"log",
|
||||
"ndk",
|
||||
"ndk-sys",
|
||||
"ndk 0.9.0",
|
||||
"ndk-sys 0.6.0+11769913",
|
||||
"objc2 0.6.4",
|
||||
"objc2-app-kit 0.3.2",
|
||||
"objc2-foundation 0.3.2",
|
||||
@@ -10447,6 +10735,16 @@ dependencies = [
|
||||
"windows-version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.54.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"
|
||||
dependencies = [
|
||||
"windows-core 0.54.0",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.61.3"
|
||||
@@ -10490,6 +10788,16 @@ dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.54.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"
|
||||
dependencies = [
|
||||
"windows-result 0.1.2",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.61.2"
|
||||
@@ -10603,6 +10911,15 @@ dependencies = [
|
||||
"windows-strings 0.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
@@ -10985,7 +11302,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"libc",
|
||||
"memmap2",
|
||||
"ndk",
|
||||
"ndk 0.9.0",
|
||||
"objc2 0.5.2",
|
||||
"objc2-app-kit 0.2.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
@@ -11296,7 +11613,7 @@ dependencies = [
|
||||
"javascriptcore-rs",
|
||||
"jni 0.21.1",
|
||||
"libc",
|
||||
"ndk",
|
||||
"ndk 0.9.0",
|
||||
"objc2 0.6.4",
|
||||
"objc2-app-kit 0.3.2",
|
||||
"objc2-core-foundation",
|
||||
|
||||
@@ -31,6 +31,7 @@ open = "5"
|
||||
reqwest = { workspace = true, features = ["multipart"] }
|
||||
resvg = "0.45"
|
||||
rfd = "0.16"
|
||||
rodio = { version = "0.20", default-features = false, features = ["symphonia-all"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
|
||||
|
||||
+20
-12
@@ -7,8 +7,8 @@ The look follows the brutalist webapp (`artcraft-services/frontend/apps/artcraft
|
||||
means near-black surfaces with white/15 hairlines and 3 px corners on controls. Type is Archivo for
|
||||
display, Inter for body text and Geist Mono for labels. The brand blue marks the primary actions.
|
||||
|
||||
Create Image and Create Video are fully built. The other tools show a "coming soon" page that
|
||||
links to the same tool in the web app.
|
||||
Create Image, Create Video and Create Audio are fully built. The other tools show a "coming soon"
|
||||
page that links to the same tool in the web app.
|
||||
|
||||
## Running
|
||||
|
||||
@@ -32,9 +32,10 @@ The workspace's default member is the Tauri app, so pass `-p artcraft_native`.
|
||||
| `backend::wire` | Lenient response shapes; new server enum values can't break decoding |
|
||||
| `backend::session` | Credentials: own file, Tauri cookie import, env config |
|
||||
| `backend::media_cache` | Thumbnails and previews: downloaded and decoded off-thread, evicted when unused |
|
||||
| `backend::audio` | One clip at a time: reference previews, feed tracks (pause, seek, volume), waveforms |
|
||||
| `models` | Model catalog from OmniGen listings, plus names, families and limits |
|
||||
| `prompt_box` | The shared prompt box: editor, reference deck, keyframes, pickers, model selector |
|
||||
| `pages` | Create Image, Create Video, Home, Library, and the shared page shell |
|
||||
| `pages` | Create Image, Video and Audio, Home, Library, and the shared page shell |
|
||||
| `feed` | Generation feed: running, failed and finished jobs, library paging, grid and list |
|
||||
| `overlays` | Lightbox, library picker, sign-in and settings dialogs |
|
||||
| `shell` | Custom title bar, sidebar flush to the window edge, content panel header |
|
||||
@@ -42,12 +43,13 @@ The workspace's default member is the Tauri app, so pass `-p artcraft_native`.
|
||||
|
||||
How the backend works:
|
||||
|
||||
- **Generation.** It goes through OmniGen: `/v1/omni_gen/{models,cost,generate}/{image,video}`.
|
||||
- **Generation.** It goes through OmniGen: `/v1/omni_gen/{models,cost,generate}/{image,video,audio}`.
|
||||
Model capabilities come from the server, so new models and options appear without an app update.
|
||||
- **Typed client functions.** Model lists, uploads, credits, login and deletes use
|
||||
`artcraft_client`.
|
||||
- **Lenient decoding.** Generate and cost requests are JSON maps, as in the Tauri app. Job polling
|
||||
(`/v1/jobs/session`), the library list, batches and prompts are decoded with lenient structs.
|
||||
(`/v1/jobs/session`), the audio model list, the library list, batches and prompts are decoded
|
||||
with lenient structs.
|
||||
- **Identity headers.** Every request sends the desktop identity headers: `Origin` and
|
||||
`User-Agent`.
|
||||
|
||||
@@ -69,23 +71,29 @@ How the backend works:
|
||||
| Live credit cost on the generate button | Done |
|
||||
| Validation: starting frame, text-only banner, limits, uploads | Done |
|
||||
| Clear all (confirms when references are attached) | Done |
|
||||
| `@Character` mentions and the Characters modal (Seedance 2.0) | Not yet |
|
||||
| `@Character` mentions and the Characters modal (create/edit/delete) | Done |
|
||||
| Provider choice (Midjourney direct, fal) | Not yet; everything runs via ArtCraft |
|
||||
| In-app video and audio playback | Opens in the system player instead |
|
||||
| Animated video previews in the feed | Still frames only |
|
||||
| In-app video playback | Done with FFmpeg on the PATH; otherwise opens in the system player |
|
||||
| Audio reference playback (click to play/stop) | Done |
|
||||
| Animated video previews in the feed (with a stills toggle) | Done |
|
||||
| Audio: style, instrumental, keep lyrics, loop | Done |
|
||||
| Audio: Beat & Key (BPM, musical key) and Tuning (sample rate, speed, volume, pitch) | Done |
|
||||
| Audio references (play, 600 s cap) or one image, never both | Done |
|
||||
|
||||
Feed and lightbox:
|
||||
|
||||
- **Feed.** Pending cards show progress, time left and a batch banner. Failed cards show the reason
|
||||
and can be dismissed. The grid is masonry, with a list view alongside. Hover actions are Recreate,
|
||||
Make Video, Share and Download. Select mode does batch downloads. The last viewed item is marked.
|
||||
- **Audio.** Audio cards play in place: a waveform (click to seek), the time and a volume flyout.
|
||||
One track plays at a time and stops when you leave the page. Audio offers Share and Download.
|
||||
- **Lightbox.** Prev/next with the arrow keys, prompt copy and details. Actions are Recreate, Make
|
||||
Video, Download, Share and Delete.
|
||||
Video, Download, Share and Delete. Audio plays in a panel with the full-size player.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Set `ARTCRAFT_SCREENSHOT=<file.png>` to save a screenshot once the window has settled.
|
||||
`ARTCRAFT_PAGE` (`home`, `create-image`, `create-video` or `library`) picks the page, and
|
||||
`ARTCRAFT_PAGE` (`home`, `create-image`, `create-video`, `create-audio` or `library`) picks the page, and
|
||||
`ARTCRAFT_SCREENSHOT_EXIT=1` closes the app afterwards.
|
||||
|
||||
## Tests
|
||||
@@ -95,8 +103,8 @@ cargo test -p artcraft_native
|
||||
```
|
||||
|
||||
There are unit tests for the catalog, feed store, wire decoding and backend helpers. Headless UI
|
||||
tests (`egui_kittest`) drive the prompt box: generate gating, Enter to generate, the pickers,
|
||||
`@` mentions and clear-all.
|
||||
tests (`egui_kittest`) drive the prompt box (generate gating, Enter to generate, the pickers, `@`
|
||||
mentions, clear-all), the Characters dialog and the audio toolbar (toggles, Beat & Key, Tuning).
|
||||
|
||||
## Fonts and brand assets
|
||||
|
||||
|
||||
+227
-95
@@ -7,6 +7,7 @@ use std::time::Duration;
|
||||
use egui::{Id, Ui};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::backend::audio::AudioPlayer;
|
||||
use crate::backend::media_cache::MediaCache;
|
||||
use crate::backend::wire::{Prompt, SessionUser};
|
||||
use crate::backend::{Backend, Event, Modality, SHARE_URL_BASE};
|
||||
@@ -14,15 +15,18 @@ use crate::feed::grid::{RatioCache, ViewMode};
|
||||
use crate::feed::store::{FeedNotice, FeedStore};
|
||||
use crate::feed::types::{FeedItem, MediaKind};
|
||||
use crate::models::Catalog;
|
||||
use crate::backend::wire::{Character, JobPhase};
|
||||
use crate::overlays::account::{LoginAction, LoginDialog, SettingsAction, settings_dialog};
|
||||
use crate::overlays::characters::{CharactersAction, CharactersModal};
|
||||
use crate::overlays::library_picker::{LibraryPicker, PickerAction};
|
||||
use crate::overlays::lightbox::{Lightbox, LightboxAction, fit_inside};
|
||||
use crate::pages::common::{self, AppRequest, Env, FeedKey};
|
||||
use crate::pages::create_audio::{AudioSettings, CreateAudioPage};
|
||||
use crate::pages::create_image::{CreateImagePage, ImageSettings};
|
||||
use crate::pages::create_video::{CreateVideoPage, VideoSettings};
|
||||
use crate::pages::other;
|
||||
use crate::prompt_box::refs::{RefKind, RefMedia};
|
||||
use crate::shell::{self, Account, Page, ShellAction};
|
||||
use crate::prompt_box::refs::{RefKind, RefMedia, References};
|
||||
use crate::shell::{self, Account, FeedToggles, Page, ShellAction};
|
||||
use crate::theme;
|
||||
use crate::ui::toast::Toasts;
|
||||
use crate::ui::{creator_icons, widgets, window};
|
||||
@@ -31,7 +35,7 @@ use crate::ui::{creator_icons, widgets, window};
|
||||
/// borrowed mutably alongside it).
|
||||
macro_rules! env {
|
||||
($app:ident, $ctx:expr) => {
|
||||
Env { ctx: $ctx.clone(), backend: &$app.backend, cache: &mut $app.cache, toasts: &mut $app.toasts, catalog: &$app.catalog, signed_in: $app.user.is_some(), enter_to_generate: $app.enter_to_generate, view_mode: $app.view_mode, ratios: &mut $app.ratios, prompts: &$app.prompts, requests: &mut $app.requests }
|
||||
Env { ctx: $ctx.clone(), backend: &$app.backend, cache: &mut $app.cache, audio: &mut $app.player, toasts: &mut $app.toasts, catalog: &$app.catalog, signed_in: $app.user.is_some(), enter_to_generate: $app.enter_to_generate, view_mode: $app.view_mode, autoplay: $app.autoplay, ratios: &mut $app.ratios, prompts: &$app.prompts, requests: &mut $app.requests, characters: &$app.characters }
|
||||
};
|
||||
}
|
||||
|
||||
@@ -41,6 +45,8 @@ const CREDITS_INTERVAL: f64 = 60.0;
|
||||
const MODEL_RETRY_INTERVAL: f64 = 15.0;
|
||||
/// List view: prompt records fetched per frame at most.
|
||||
const PROMPT_FETCHES_PER_FRAME: usize = 8;
|
||||
/// Every create page's model listing.
|
||||
const MODALITIES: [Modality; 3] = [Modality::Image, Modality::Video, Modality::Audio];
|
||||
|
||||
/// Preferences that survive restarts.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
@@ -49,20 +55,23 @@ struct Saved {
|
||||
page: Page,
|
||||
enter_to_generate: bool,
|
||||
view_mode: ViewMode,
|
||||
autoplay: bool,
|
||||
sidebar_open: bool,
|
||||
image: ImageSettings,
|
||||
video: VideoSettings,
|
||||
audio: AudioSettings,
|
||||
}
|
||||
|
||||
impl Default for Saved {
|
||||
fn default() -> Self {
|
||||
Self { page: Page::CreateImage, enter_to_generate: false, view_mode: ViewMode::Grid, sidebar_open: true, image: ImageSettings { count: 1, ..Default::default() }, video: VideoSettings::default() }
|
||||
Self { page: Page::CreateImage, enter_to_generate: false, view_mode: ViewMode::Grid, autoplay: true, sidebar_open: true, image: ImageSettings { count: 1, ..Default::default() }, video: VideoSettings::default(), audio: AudioSettings::default() }
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ArtcraftApp {
|
||||
backend: Backend,
|
||||
cache: MediaCache,
|
||||
player: AudioPlayer,
|
||||
toasts: Toasts,
|
||||
catalog: Catalog,
|
||||
user: Option<SessionUser>,
|
||||
@@ -70,9 +79,11 @@ pub struct ArtcraftApp {
|
||||
page: Page,
|
||||
enter_to_generate: bool,
|
||||
view_mode: ViewMode,
|
||||
autoplay: bool,
|
||||
sidebar_open: bool,
|
||||
image: CreateImagePage,
|
||||
video: CreateVideoPage,
|
||||
audio: CreateAudioPage,
|
||||
library: FeedStore,
|
||||
ratios: RatioCache,
|
||||
/// Prompt texts by prompt token (list view, lightbox).
|
||||
@@ -86,6 +97,10 @@ pub struct ArtcraftApp {
|
||||
lightbox: Option<Lightbox>,
|
||||
picker: Option<LibraryPicker>,
|
||||
login: Option<LoginDialog>,
|
||||
characters_modal: Option<CharactersModal>,
|
||||
characters: Vec<Character>,
|
||||
/// Characters being made: (job token, name).
|
||||
pending_characters: Vec<(String, String)>,
|
||||
settings_open: bool,
|
||||
preview: Option<String>,
|
||||
requests: Vec<AppRequest>,
|
||||
@@ -105,10 +120,12 @@ impl ArtcraftApp {
|
||||
}
|
||||
let backend = Backend::new(cc.egui_ctx.clone());
|
||||
let cache = MediaCache::new(backend.runtime(), backend.http());
|
||||
backend.load_models(Modality::Image);
|
||||
backend.load_models(Modality::Video);
|
||||
let player = AudioPlayer::new(backend.runtime(), backend.http());
|
||||
for modality in MODALITIES {
|
||||
backend.load_models(modality);
|
||||
}
|
||||
backend.refresh_session();
|
||||
Self { cache, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, settings_open: false, preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend }
|
||||
Self { cache, player, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, autoplay: saved.autoplay, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), audio: CreateAudioPage::new(saved.audio), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, characters_modal: None, characters: Vec::new(), pending_characters: Vec::new(), settings_open: false, preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend }
|
||||
}
|
||||
|
||||
// --- Events from the backend -------------------------------------------------------------
|
||||
@@ -119,13 +136,16 @@ impl ArtcraftApp {
|
||||
let changed = self.user.as_ref().map(|u| &u.username) != user.as_ref().map(|u| &u.username);
|
||||
self.user = user;
|
||||
if changed {
|
||||
self.image.feed.reset();
|
||||
self.video.feed.reset();
|
||||
self.library.reset();
|
||||
for feed in self.feeds_mut() {
|
||||
feed.reset();
|
||||
}
|
||||
self.next_poll = 0.0;
|
||||
}
|
||||
if self.user.is_some() {
|
||||
self.login = None;
|
||||
if changed {
|
||||
self.backend.load_characters();
|
||||
}
|
||||
} else {
|
||||
self.credits = None;
|
||||
if let Some(login) = &mut self.login {
|
||||
@@ -149,6 +169,7 @@ impl ArtcraftApp {
|
||||
Event::Models(modality, Ok(models)) => match modality {
|
||||
Modality::Image => self.catalog.set_image(models),
|
||||
Modality::Video => self.catalog.set_video(models),
|
||||
Modality::Audio => self.catalog.set_audio(models),
|
||||
},
|
||||
Event::Models(_, Err(message)) => {
|
||||
if self.models_retry_at.is_none() {
|
||||
@@ -157,34 +178,65 @@ impl ArtcraftApp {
|
||||
self.models_retry_at = Some(ctx.input(|i| i.time) + MODEL_RETRY_INTERVAL);
|
||||
},
|
||||
Event::Cost { key, credits } => {
|
||||
self.image.cost.apply(&key, credits);
|
||||
self.video.cost.apply(&key, credits);
|
||||
for cost in [&mut self.image.cost, &mut self.video.cost, &mut self.audio.cost] {
|
||||
cost.apply(&key, credits);
|
||||
}
|
||||
},
|
||||
Event::Enqueued { modality, job_tokens, meta } => {
|
||||
let mut env = env!(self, ctx);
|
||||
match modality {
|
||||
Modality::Image => self.image.on_enqueued(&mut env, &job_tokens, &meta),
|
||||
Modality::Video => self.video.on_enqueued(&mut env, &job_tokens, &meta),
|
||||
Modality::Audio => self.audio.on_enqueued(&mut env, &job_tokens, &meta),
|
||||
}
|
||||
self.next_poll = ctx.input(|i| i.time) + 2.0;
|
||||
},
|
||||
Event::EnqueueFailed { modality, message } => {
|
||||
Event::EnqueueFailed { modality, message, status } => {
|
||||
let mut env = env!(self, ctx);
|
||||
match modality {
|
||||
Modality::Image => self.image.on_enqueue_failed(&mut env, &message),
|
||||
Modality::Video => self.video.on_enqueue_failed(&mut env, &message),
|
||||
Modality::Audio => self.audio.on_enqueue_failed(&mut env, &message, status),
|
||||
}
|
||||
},
|
||||
Event::Uploaded { ref_id, result } => {
|
||||
if let Some(modal) = &mut self.characters_modal {
|
||||
match modal.apply_upload(ref_id, result.clone()) {
|
||||
Some(Err(err)) => {
|
||||
self.toasts.error(format!("Failed to upload the reference image: {err}"));
|
||||
return;
|
||||
},
|
||||
Some(Ok(())) => return,
|
||||
None => {},
|
||||
}
|
||||
}
|
||||
if self.image.refs.find_mut(ref_id).is_some() {
|
||||
common::apply_upload(&mut self.toasts, &mut self.image.refs, ref_id, result);
|
||||
} else if self.audio.refs.find_mut(ref_id).is_some() {
|
||||
if common::apply_upload(&mut self.toasts, &mut self.audio.refs, ref_id, result) {
|
||||
self.audio.check_durations(&mut self.toasts, ref_id);
|
||||
}
|
||||
} else if common::apply_upload(&mut self.toasts, &mut self.video.refs, ref_id, result) {
|
||||
let mut env = env!(self, ctx);
|
||||
self.video.check_durations(&mut env, ref_id);
|
||||
}
|
||||
},
|
||||
Event::Jobs(jobs) => {
|
||||
for (kind, notices) in [(MediaKind::Image, self.image.feed.apply_jobs(MediaKind::Image, &jobs, &self.catalog)), (MediaKind::Video, self.video.feed.apply_jobs(MediaKind::Video, &jobs, &self.catalog))] {
|
||||
// Characters being made finish as jobs too.
|
||||
let before = self.pending_characters.len();
|
||||
let toasts = &mut self.toasts;
|
||||
self.pending_characters.retain(|(job, name)| match jobs.iter().find(|j| &j.job_token == job).map(|j| j.status.phase()) {
|
||||
Some(JobPhase::Succeeded) => false,
|
||||
Some(JobPhase::Failed) => {
|
||||
toasts.error(format!("Character \u{201c}{name}\u{201d} failed to create"));
|
||||
false
|
||||
},
|
||||
_ => true,
|
||||
});
|
||||
if self.pending_characters.len() != before {
|
||||
self.backend.load_characters();
|
||||
}
|
||||
for (kind, notices) in [(MediaKind::Image, self.image.feed.apply_jobs(MediaKind::Image, &jobs, &self.catalog)), (MediaKind::Video, self.video.feed.apply_jobs(MediaKind::Video, &jobs, &self.catalog)), (MediaKind::Audio, self.audio.feed.apply_jobs(MediaKind::Audio, &jobs, &self.catalog))] {
|
||||
for notice in notices {
|
||||
match notice {
|
||||
FeedNotice::Completed => {
|
||||
@@ -200,33 +252,30 @@ impl ArtcraftApp {
|
||||
}
|
||||
}
|
||||
},
|
||||
Event::LibraryPage { key, page, items, total_pages } => match key.as_str() {
|
||||
"picker" => {
|
||||
// The picker's pages come back under its own name.
|
||||
Event::LibraryPage { key, page, items, total_pages } => match FeedKey::from_name(&key) {
|
||||
Some(feed) => self.feed_mut(feed).apply_library_page(feed.kind(), page, &items, total_pages),
|
||||
None => {
|
||||
if let Some(p) = &mut self.picker {
|
||||
p.apply_page(page, items, total_pages);
|
||||
}
|
||||
},
|
||||
"image" => self.image.feed.apply_library_page(MediaKind::Image, page, &items, total_pages),
|
||||
"video" => self.video.feed.apply_library_page(MediaKind::Video, page, &items, total_pages),
|
||||
_ => self.library.apply_library_page(MediaKind::Image, page, &items, total_pages),
|
||||
},
|
||||
Event::LibraryFailed { key, message } => {
|
||||
match key.as_str() {
|
||||
"picker" => {
|
||||
match FeedKey::from_name(&key) {
|
||||
Some(feed) => self.feed_mut(feed).library_failed(),
|
||||
None => {
|
||||
if let Some(p) = &mut self.picker {
|
||||
p.loading = false;
|
||||
p.has_more = false;
|
||||
}
|
||||
},
|
||||
"image" => self.image.feed.library_failed(),
|
||||
"video" => self.video.feed.library_failed(),
|
||||
_ => self.library.library_failed(),
|
||||
}
|
||||
self.toasts.error(format!("Couldn't load your library: {message}"));
|
||||
},
|
||||
Event::Batch { job_token, items } => match self.batch_kinds.remove(&job_token) {
|
||||
Some(MediaKind::Video) => self.video.feed.apply_batch(MediaKind::Video, &items),
|
||||
_ => self.image.feed.apply_batch(MediaKind::Image, &items),
|
||||
Event::Batch { job_token, items } => {
|
||||
let kind = self.batch_kinds.remove(&job_token).unwrap_or(MediaKind::Image);
|
||||
self.feed_mut(FeedKey::of(kind)).apply_batch(kind, &items);
|
||||
},
|
||||
Event::Prompt(prompt) => {
|
||||
if let Some(text) = &prompt.maybe_positive_prompt {
|
||||
@@ -239,8 +288,15 @@ impl ArtcraftApp {
|
||||
}
|
||||
self.prompt_records.insert(prompt.token.clone(), prompt);
|
||||
},
|
||||
Event::Characters(list) => self.characters = list,
|
||||
Event::CharacterCreating { job_token, name } => {
|
||||
self.toasts.success(format!("Character \u{201c}{name}\u{201d} is being created"));
|
||||
self.pending_characters.push((job_token, name));
|
||||
self.next_poll = 0.0;
|
||||
},
|
||||
Event::CharacterChanged => self.backend.load_characters(),
|
||||
Event::Deleted(token) => {
|
||||
for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] {
|
||||
for feed in self.feeds_mut() {
|
||||
feed.remove_item(&token);
|
||||
}
|
||||
if self.lightbox.as_ref().is_some_and(|l| l.token == token) {
|
||||
@@ -274,7 +330,7 @@ impl ArtcraftApp {
|
||||
self.open_login();
|
||||
return;
|
||||
};
|
||||
let refs = if page == MediaKind::Image { &self.image.refs } else { &self.video.refs };
|
||||
let refs = self.refs(page);
|
||||
let attached: HashSet<String> = refs.list(kind).iter().chain(refs.first_frame.iter()).chain(refs.last_frame.iter()).filter_map(|r| r.token.clone()).collect();
|
||||
let mut picker = LibraryPicker::new(kind, slot, page, max, attached);
|
||||
picker.loading = true;
|
||||
@@ -287,8 +343,7 @@ impl ArtcraftApp {
|
||||
};
|
||||
let image = RefMedia::from_library(RefKind::Image, item.token.clone(), item.thumbnail.clone(), Some(item.full_url.clone()), 0.0);
|
||||
self.video.set_start_image(image);
|
||||
self.lightbox = None;
|
||||
self.page = Page::CreateVideo;
|
||||
self.navigate(Page::CreateVideo);
|
||||
},
|
||||
AppRequest::Recreate { kind, token } => {
|
||||
let Some(prompt_token) = self.find_item(&token).and_then(|i| i.prompt_token.clone()) else {
|
||||
@@ -314,17 +369,27 @@ impl ArtcraftApp {
|
||||
},
|
||||
AppRequest::Download(tokens) => self.download(&tokens),
|
||||
AppRequest::Preview(url) => self.preview = Some(url),
|
||||
AppRequest::ToggleAudio { ref_id, url } => self.player.toggle(ctx, &format!("ref:{ref_id}"), &url),
|
||||
AppRequest::OpenCharacters => {
|
||||
if self.user.is_none() {
|
||||
self.open_login();
|
||||
return;
|
||||
}
|
||||
self.backend.load_characters();
|
||||
self.characters_modal = Some(CharactersModal::new());
|
||||
},
|
||||
AppRequest::PlayExternally(url) => {
|
||||
if let Err(err) = open::that(&url) {
|
||||
self.toasts.error(format!("Couldn't open the file: {err}"));
|
||||
}
|
||||
},
|
||||
AppRequest::LoadMoreLibrary(key) => {
|
||||
let Some(user) = &self.user else {
|
||||
return;
|
||||
};
|
||||
let (feed, name) = match key {
|
||||
FeedKey::Image => (&mut self.image.feed, "image"),
|
||||
FeedKey::Video => (&mut self.video.feed, "video"),
|
||||
FeedKey::Library => (&mut self.library, "library"),
|
||||
};
|
||||
if let Some(page) = feed.next_library_page() {
|
||||
self.backend.load_library(name.to_owned(), user.username.clone(), key.media_classes(), page);
|
||||
let username = user.username.clone();
|
||||
if let Some(page) = self.feed_mut(key).next_library_page() {
|
||||
self.backend.load_library(key.name().to_owned(), username, key.media_classes(), page);
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -334,14 +399,15 @@ impl ArtcraftApp {
|
||||
match kind {
|
||||
MediaKind::Image => {
|
||||
self.image.apply_prompt(prompt);
|
||||
self.page = Page::CreateImage;
|
||||
self.navigate(Page::CreateImage);
|
||||
},
|
||||
MediaKind::Video => {
|
||||
self.video.apply_prompt(prompt);
|
||||
self.page = Page::CreateVideo;
|
||||
self.navigate(Page::CreateVideo);
|
||||
},
|
||||
// Audio isn't recreated yet (no Recreate button offers it).
|
||||
MediaKind::Audio => {},
|
||||
}
|
||||
self.lightbox = None;
|
||||
}
|
||||
|
||||
fn download(&mut self, tokens: &[String]) {
|
||||
@@ -359,7 +425,7 @@ impl ArtcraftApp {
|
||||
for item in many {
|
||||
self.backend.save_url(item.full_url.clone(), dir.join(file_name_for(item)));
|
||||
}
|
||||
for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] {
|
||||
for feed in self.feeds_mut() {
|
||||
feed.selecting = false;
|
||||
feed.selected.clear();
|
||||
}
|
||||
@@ -369,7 +435,52 @@ impl ArtcraftApp {
|
||||
}
|
||||
|
||||
fn find_item(&self, token: &str) -> Option<&FeedItem> {
|
||||
self.image.feed.find(token).or_else(|| self.video.feed.find(token)).or_else(|| self.library.find(token))
|
||||
self.feeds().into_iter().find_map(|f| f.find(token))
|
||||
}
|
||||
|
||||
fn feeds(&self) -> [&FeedStore; 4] {
|
||||
[&self.image.feed, &self.video.feed, &self.audio.feed, &self.library]
|
||||
}
|
||||
|
||||
fn feeds_mut(&mut self) -> [&mut FeedStore; 4] {
|
||||
[&mut self.image.feed, &mut self.video.feed, &mut self.audio.feed, &mut self.library]
|
||||
}
|
||||
|
||||
fn feed(&self, key: FeedKey) -> &FeedStore {
|
||||
match key {
|
||||
FeedKey::Image => &self.image.feed,
|
||||
FeedKey::Video => &self.video.feed,
|
||||
FeedKey::Audio => &self.audio.feed,
|
||||
FeedKey::Library => &self.library,
|
||||
}
|
||||
}
|
||||
|
||||
fn feed_mut(&mut self, key: FeedKey) -> &mut FeedStore {
|
||||
match key {
|
||||
FeedKey::Image => &mut self.image.feed,
|
||||
FeedKey::Video => &mut self.video.feed,
|
||||
FeedKey::Audio => &mut self.audio.feed,
|
||||
FeedKey::Library => &mut self.library,
|
||||
}
|
||||
}
|
||||
|
||||
/// The references of the page that makes `kind`.
|
||||
fn refs(&self, kind: MediaKind) -> &References {
|
||||
match kind {
|
||||
MediaKind::Image => &self.image.refs,
|
||||
MediaKind::Video => &self.video.refs,
|
||||
MediaKind::Audio => &self.audio.refs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens `page`, closing the lightbox. Leaving a page stops its audio, like the webapp's
|
||||
/// players unmounting.
|
||||
fn navigate(&mut self, page: Page) {
|
||||
if page != self.page {
|
||||
self.player.stop();
|
||||
}
|
||||
self.page = page;
|
||||
self.lightbox = None;
|
||||
}
|
||||
|
||||
fn request_prompt(&mut self, prompt_token: Option<String>) {
|
||||
@@ -386,12 +497,13 @@ impl ArtcraftApp {
|
||||
}
|
||||
}
|
||||
|
||||
/// The feed the top bar's toggles act on.
|
||||
fn current_feed(&mut self) -> Option<&mut FeedStore> {
|
||||
/// The feed the page shows (and the top bar's toggles act on).
|
||||
fn page_feed(&self) -> Option<FeedKey> {
|
||||
match self.page {
|
||||
Page::CreateImage => Some(&mut self.image.feed),
|
||||
Page::CreateVideo => Some(&mut self.video.feed),
|
||||
Page::Library => Some(&mut self.library),
|
||||
Page::CreateImage => Some(FeedKey::Image),
|
||||
Page::CreateVideo => Some(FeedKey::Video),
|
||||
Page::CreateAudio => Some(FeedKey::Audio),
|
||||
Page::Library => Some(FeedKey::Library),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -410,11 +522,10 @@ impl ArtcraftApp {
|
||||
}
|
||||
if self.models_retry_at.is_some_and(|t| now >= t) {
|
||||
self.models_retry_at = None;
|
||||
if self.catalog.image.is_empty() {
|
||||
self.backend.load_models(Modality::Image);
|
||||
}
|
||||
if self.catalog.video.is_empty() {
|
||||
self.backend.load_models(Modality::Video);
|
||||
for (modality, loaded) in MODALITIES.into_iter().zip([&self.catalog.image, &self.catalog.video, &self.catalog.audio]) {
|
||||
if loaded.is_empty() {
|
||||
self.backend.load_models(modality);
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx.request_repaint_after(Duration::from_secs(1));
|
||||
@@ -441,10 +552,7 @@ impl ArtcraftApp {
|
||||
|
||||
fn apply_shell_action(&mut self, action: ShellAction) {
|
||||
match action {
|
||||
ShellAction::Navigate(page) => {
|
||||
self.page = page;
|
||||
self.lightbox = None;
|
||||
},
|
||||
ShellAction::Navigate(page) => self.navigate(page),
|
||||
ShellAction::ToggleSidebar => self.sidebar_open = !self.sidebar_open,
|
||||
ShellAction::SignIn => self.open_login(),
|
||||
ShellAction::SignOut => {
|
||||
@@ -453,8 +561,10 @@ impl ArtcraftApp {
|
||||
},
|
||||
ShellAction::OpenSettings => self.settings_open = true,
|
||||
ShellAction::SetViewMode(mode) => self.view_mode = mode,
|
||||
ShellAction::SetAutoplay(on) => self.autoplay = on,
|
||||
ShellAction::ToggleSelect => {
|
||||
if let Some(feed) = self.current_feed() {
|
||||
if let Some(key) = self.page_feed() {
|
||||
let feed = self.feed_mut(key);
|
||||
feed.selecting = !feed.selecting;
|
||||
feed.selected.clear();
|
||||
}
|
||||
@@ -468,13 +578,10 @@ impl ArtcraftApp {
|
||||
let ctx = ui.ctx().clone();
|
||||
let gap = shell::PANEL_GAP;
|
||||
let margin = egui::Margin { left: if self.sidebar_open { 0 } else { gap }, right: gap, top: 0, bottom: gap };
|
||||
let toggles = match self.page {
|
||||
Page::CreateImage => Some((self.view_mode, self.image.feed.selecting)),
|
||||
Page::CreateVideo => Some((self.view_mode, self.video.feed.selecting)),
|
||||
Page::Library => Some((self.view_mode, self.library.selecting)),
|
||||
_ => None,
|
||||
};
|
||||
// Only feeds with videos offer the preview autoplay toggle.
|
||||
let toggles = self.page_feed().map(|key| FeedToggles { mode: self.view_mode, selecting: self.feed(key).selecting, autoplay: matches!(key, FeedKey::Video | FeedKey::Library).then_some(self.autoplay) });
|
||||
let mut header_action = None;
|
||||
let mut open_page = None;
|
||||
egui::CentralPanel::default().frame(egui::Frame::NONE.fill(theme::CHROME).inner_margin(margin)).show(ui, |ui| {
|
||||
let panel = egui::Frame::new().fill(theme::BG).stroke(theme::hairline()).corner_radius(shell::PANEL_RADIUS);
|
||||
panel.show(ui, |ui| {
|
||||
@@ -485,13 +592,10 @@ impl ArtcraftApp {
|
||||
egui::CentralPanel::default().frame(egui::Frame::NONE).show(ui, |ui| {
|
||||
let mut env = env!(self, ctx);
|
||||
match self.page {
|
||||
Page::Home => {
|
||||
if let Some(page) = other::home(ui, env.signed_in, env.requests) {
|
||||
self.page = page;
|
||||
}
|
||||
},
|
||||
Page::Home => open_page = other::home(ui, env.signed_in, env.requests),
|
||||
Page::CreateImage => self.image.ui(ui, &mut env),
|
||||
Page::CreateVideo => self.video.ui(ui, &mut env),
|
||||
Page::CreateAudio => self.audio.ui(ui, &mut env),
|
||||
Page::Library => other::library(ui, &mut env, &mut self.library),
|
||||
page => other::coming_soon(ui, page),
|
||||
}
|
||||
@@ -501,39 +605,33 @@ impl ArtcraftApp {
|
||||
if let Some(action) = header_action {
|
||||
self.apply_shell_action(action);
|
||||
}
|
||||
self.prefetch_list_prompts();
|
||||
if let Some(page) = open_page {
|
||||
self.navigate(page);
|
||||
}
|
||||
self.prefetch_prompts();
|
||||
}
|
||||
|
||||
/// The list view shows prompt texts: fetch the records for the visible feed, a few at a time.
|
||||
fn prefetch_list_prompts(&mut self) {
|
||||
if self.view_mode != ViewMode::List {
|
||||
/// The list view and audio cards show prompt texts: fetch the records for the visible feed, a
|
||||
/// few at a time.
|
||||
fn prefetch_prompts(&mut self) {
|
||||
let Some(key) = self.page_feed() else {
|
||||
return;
|
||||
}
|
||||
let feed = match self.page {
|
||||
Page::CreateImage => &self.image.feed,
|
||||
Page::CreateVideo => &self.video.feed,
|
||||
Page::Library => &self.library,
|
||||
_ => return,
|
||||
};
|
||||
let wanted: Vec<String> = feed.items.iter().filter_map(|i| i.prompt_token.clone()).filter(|t| !self.requested_prompts.contains(t)).take(PROMPT_FETCHES_PER_FRAME).collect();
|
||||
let list = self.view_mode == ViewMode::List;
|
||||
let wanted: Vec<String> = self.feed(key).items.iter().filter(|i| list || i.kind == MediaKind::Audio).filter_map(|i| i.prompt_token.clone()).filter(|t| !self.requested_prompts.contains(t)).take(PROMPT_FETCHES_PER_FRAME).collect();
|
||||
for token in wanted {
|
||||
self.request_prompt(Some(token));
|
||||
}
|
||||
}
|
||||
|
||||
fn overlays(&mut self, ctx: &egui::Context) {
|
||||
if let Some(lb) = &mut self.lightbox {
|
||||
let feed = match lb.kind {
|
||||
_ if self.page == Page::Library => &self.library,
|
||||
MediaKind::Image => &self.image.feed,
|
||||
MediaKind::Video => &self.video.feed,
|
||||
};
|
||||
let order = feed.ordered_tokens();
|
||||
let item = self.image.feed.find(&lb.token).or_else(|| self.video.feed.find(&lb.token)).or_else(|| self.library.find(&lb.token)).cloned();
|
||||
match item {
|
||||
if let Some((kind, token)) = self.lightbox.as_ref().map(|lb| (lb.kind, lb.token.clone())) {
|
||||
let key = if self.page == Page::Library { FeedKey::Library } else { FeedKey::of(kind) };
|
||||
let order = self.feed(key).ordered_tokens();
|
||||
match self.find_item(&token).cloned() {
|
||||
Some(item) => {
|
||||
let prompt = item.prompt_token.as_ref().and_then(|t| self.prompt_records.get(t));
|
||||
let actions = lb.show(ctx, &item, &order, prompt, &self.catalog, &mut self.cache);
|
||||
let actions = self.lightbox.as_mut().map(|lb| lb.show(ctx, &item, &order, prompt, &self.catalog, &mut self.cache, &mut self.player)).unwrap_or_default();
|
||||
for action in actions {
|
||||
self.lightbox_action(ctx, action, &item);
|
||||
}
|
||||
@@ -555,8 +653,11 @@ impl ArtcraftApp {
|
||||
},
|
||||
PickerAction::Confirm(picks) => {
|
||||
if let Some(p) = self.picker.take() {
|
||||
let refs = if p.page == MediaKind::Image { &mut self.image.refs } else { &mut self.video.refs };
|
||||
common::attach_from_library(refs, p.kind, p.slot, picks);
|
||||
match p.page {
|
||||
MediaKind::Image => common::attach_from_library(&mut self.image.refs, p.kind, p.slot, picks),
|
||||
MediaKind::Video => common::attach_from_library(&mut self.video.refs, p.kind, p.slot, picks),
|
||||
MediaKind::Audio => self.audio.attach_from_library(&mut self.toasts, &self.catalog, p.kind, picks),
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -584,6 +685,27 @@ impl ArtcraftApp {
|
||||
None => {},
|
||||
}
|
||||
}
|
||||
if let Some(modal) = &mut self.characters_modal {
|
||||
let pending: Vec<String> = self.pending_characters.iter().map(|(_, n)| n.clone()).collect();
|
||||
for action in modal.show(ctx, &self.characters, &pending, &mut self.cache) {
|
||||
match action {
|
||||
CharactersAction::Close => self.characters_modal = None,
|
||||
CharactersAction::Select { token, name } => {
|
||||
self.video.mention_character(&name, &token);
|
||||
self.characters_modal = None;
|
||||
},
|
||||
CharactersAction::Upload { ref_id, path } => self.backend.upload_file(ref_id, RefKind::Image, path),
|
||||
CharactersAction::Create { image_token, name, description } => self.backend.create_character(image_token, name, description),
|
||||
CharactersAction::Edit { token, name, description } => self.backend.edit_character(token, name, description),
|
||||
CharactersAction::Delete { token, name } => self.backend.delete_character(token, name),
|
||||
CharactersAction::Preview(url) => self.preview = Some(url),
|
||||
CharactersAction::Toast(message) => self.toasts.error(message),
|
||||
}
|
||||
if self.characters_modal.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if self.settings_open {
|
||||
let username = self.user.as_ref().map(|u| u.username.clone());
|
||||
match settings_dialog(ctx, &mut self.enter_to_generate, username.as_deref(), self.credits, self.backend.data_root()) {
|
||||
@@ -625,7 +747,7 @@ impl ArtcraftApp {
|
||||
if let Some(next) = self.find_item(&token).cloned() {
|
||||
self.request_prompt(next.prompt_token);
|
||||
}
|
||||
for feed in [&mut self.image.feed, &mut self.video.feed, &mut self.library] {
|
||||
for feed in self.feeds_mut() {
|
||||
if feed.find(&token).is_some() {
|
||||
feed.last_viewed = Some(token.clone());
|
||||
}
|
||||
@@ -663,6 +785,11 @@ impl eframe::App for ArtcraftApp {
|
||||
fn ui(&mut self, ui: &mut Ui, _frame: &mut eframe::Frame) {
|
||||
let ctx = ui.ctx().clone();
|
||||
self.cache.poll(&ctx);
|
||||
if let Some(err) = self.player.poll() {
|
||||
self.toasts.error(err);
|
||||
}
|
||||
let playing = [&self.video.refs, &self.image.refs].iter().flat_map(|r| r.audios.iter()).map(|r| r.id).find(|id| self.player.is_playing(&format!("ref:{id}")));
|
||||
crate::prompt_box::deck::set_playing_audio(&ctx, playing);
|
||||
self.shell(ui);
|
||||
self.page_ui(ui);
|
||||
for request in std::mem::take(&mut self.requests) {
|
||||
@@ -677,7 +804,7 @@ impl eframe::App for ArtcraftApp {
|
||||
}
|
||||
|
||||
fn save(&mut self, storage: &mut dyn eframe::Storage) {
|
||||
let saved = Saved { page: self.page, enter_to_generate: self.enter_to_generate, view_mode: self.view_mode, sidebar_open: self.sidebar_open, image: self.image.settings.clone(), video: self.video.settings.clone() };
|
||||
let saved = Saved { page: self.page, enter_to_generate: self.enter_to_generate, view_mode: self.view_mode, autoplay: self.autoplay, sidebar_open: self.sidebar_open, image: self.image.settings.clone(), video: self.video.settings.clone(), audio: self.audio.settings.clone() };
|
||||
eframe::set_value(storage, STORAGE_KEY, &saved);
|
||||
}
|
||||
|
||||
@@ -733,6 +860,11 @@ impl ScreenshotRequest {
|
||||
|
||||
/// `artcraft_<token>.<ext>`, the extension taken from the URL.
|
||||
fn file_name_for(item: &FeedItem) -> String {
|
||||
let ext = item.full_url.rsplit('.').next().filter(|e| e.len() <= 4 && e.chars().all(|c| c.is_ascii_alphanumeric())).unwrap_or(if item.kind == MediaKind::Video { "mp4" } else { "png" });
|
||||
let fallback = match item.kind {
|
||||
MediaKind::Image => "png",
|
||||
MediaKind::Video => "mp4",
|
||||
MediaKind::Audio => "mp3",
|
||||
};
|
||||
let ext = item.full_url.rsplit('.').next().filter(|e| e.len() <= 4 && e.chars().all(|c| c.is_ascii_alphanumeric())).unwrap_or(fallback);
|
||||
format!("artcraft_{}.{ext}", item.token)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
//! Audio, one clip at a time (`audio-playback-controller`): reference cards play and stop on
|
||||
//! click (the webapp's `DeckCard`), feed tracks pause, resume and seek (`WaveformAudioPlayer`).
|
||||
//! Tracks also get waveforms, decoded off the UI thread. The output device opens on first use;
|
||||
//! files download in the background.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use log::warn;
|
||||
use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink, Source};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// The webapp's reference preview volume.
|
||||
const PREVIEW_VOLUME: f32 = 0.2;
|
||||
/// Peaks kept per waveform (resampled to however many bars are drawn).
|
||||
const PEAK_COUNT: usize = 256;
|
||||
/// Frames folded into one peak while decoding.
|
||||
const WINDOW_FRAMES: usize = 1024;
|
||||
/// Waveforms decoding at once (each downloads its whole file).
|
||||
const MAX_CONCURRENT_WAVEFORMS: usize = 2;
|
||||
|
||||
pub struct AudioPlayer {
|
||||
rt: tokio::runtime::Handle,
|
||||
http: reqwest::Client,
|
||||
/// Opened on first play (and kept: closing it would stop the sound).
|
||||
output: Option<(OutputStream, OutputStreamHandle)>,
|
||||
sink: Option<Sink>,
|
||||
/// What's playing (or loading).
|
||||
current: Option<Clip>,
|
||||
tx: Sender<Loaded>,
|
||||
rx: Receiver<Loaded>,
|
||||
waveforms: HashMap<String, WaveformLookup>,
|
||||
limiter: Arc<Semaphore>,
|
||||
/// The tracks' volume slider, 0–1.
|
||||
track_volume: f32,
|
||||
}
|
||||
|
||||
/// A track's shape: peaks (0–1, loudest = 1) and its length in seconds.
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct Waveform {
|
||||
pub peaks: Vec<f32>,
|
||||
pub duration: f32,
|
||||
}
|
||||
|
||||
/// What [`AudioPlayer::waveform`] knows about a track.
|
||||
#[derive(Clone)]
|
||||
pub enum WaveformLookup {
|
||||
Loading,
|
||||
Ready(Arc<Waveform>),
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Where the current track is.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Playback {
|
||||
/// Still downloading.
|
||||
pub loading: bool,
|
||||
pub paused: bool,
|
||||
/// Seconds in.
|
||||
pub position: f32,
|
||||
}
|
||||
|
||||
/// The clip that's playing, by the caller's key (a track's key is its URL).
|
||||
struct Clip {
|
||||
key: String,
|
||||
track: bool,
|
||||
/// Where to start once it has downloaded (a seek before it loaded).
|
||||
start_at: f32,
|
||||
}
|
||||
|
||||
enum Loaded {
|
||||
Clip { key: String, bytes: Result<Arc<[u8]>, String> },
|
||||
Waveform { url: String, waveform: Option<Waveform> },
|
||||
}
|
||||
|
||||
impl AudioPlayer {
|
||||
pub fn new(rt: tokio::runtime::Handle, http: reqwest::Client) -> Self {
|
||||
let (tx, rx) = channel();
|
||||
Self { rt, http, output: None, sink: None, current: None, tx, rx, waveforms: HashMap::new(), limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_WAVEFORMS)), track_volume: 1.0 }
|
||||
}
|
||||
|
||||
/// Starts a reference clip under `key`, or stops it if it's what's playing.
|
||||
pub fn toggle(&mut self, ctx: &egui::Context, key: &str, url: &str) {
|
||||
if self.is_playing(key) {
|
||||
self.stop();
|
||||
} else {
|
||||
self.start(ctx, key, url, false, 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Plays or pauses a feed track.
|
||||
pub fn toggle_track(&mut self, ctx: &egui::Context, url: &str) {
|
||||
match (&self.sink, self.is_playing(url)) {
|
||||
(Some(sink), true) if sink.is_paused() => sink.play(),
|
||||
(Some(sink), true) => sink.pause(),
|
||||
// Still downloading: a second click cancels.
|
||||
(None, true) => self.stop(),
|
||||
_ => self.start(ctx, url, url, true, 0.0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Jumps a feed track to `secs` (starting it there if it isn't the current one).
|
||||
pub fn seek_track(&mut self, ctx: &egui::Context, url: &str, secs: f32) {
|
||||
if !self.is_playing(url) {
|
||||
self.start(ctx, url, url, true, secs);
|
||||
return;
|
||||
}
|
||||
match &self.sink {
|
||||
Some(sink) => {
|
||||
if let Err(err) = sink.try_seek(Duration::from_secs_f32(secs.max(0.0))) {
|
||||
warn!("Audio seek failed: {err}");
|
||||
}
|
||||
sink.play();
|
||||
},
|
||||
None => {
|
||||
if let Some(clip) = &mut self.current {
|
||||
clip.start_at = secs;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stop(&mut self) {
|
||||
if let Some(sink) = self.sink.take() {
|
||||
sink.stop();
|
||||
}
|
||||
self.current = None;
|
||||
}
|
||||
|
||||
/// Whether `key` is playing, paused or about to play.
|
||||
pub fn is_playing(&self, key: &str) -> bool {
|
||||
self.current.as_ref().is_some_and(|c| c.key == key)
|
||||
}
|
||||
|
||||
/// Where the track at `url` is, when it's the current one.
|
||||
pub fn track(&self, url: &str) -> Option<Playback> {
|
||||
let clip = self.current.as_ref().filter(|c| c.key == url)?;
|
||||
Some(match &self.sink {
|
||||
Some(sink) => Playback { loading: false, paused: sink.is_paused(), position: sink.get_pos().as_secs_f32() },
|
||||
None => Playback { loading: true, paused: false, position: clip.start_at },
|
||||
})
|
||||
}
|
||||
|
||||
pub fn track_volume(&self) -> f32 {
|
||||
self.track_volume
|
||||
}
|
||||
|
||||
pub fn set_track_volume(&mut self, volume: f32) {
|
||||
self.track_volume = volume.clamp(0.0, 1.0);
|
||||
if let (Some(sink), Some(clip)) = (&self.sink, &self.current) {
|
||||
if clip.track {
|
||||
sink.set_volume(self.track_volume);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The waveform of the track at `url`, downloading and decoding it the first time it's asked
|
||||
/// for.
|
||||
pub fn waveform(&mut self, ctx: &egui::Context, url: &str) -> WaveformLookup {
|
||||
if let Some(lookup) = self.waveforms.get(url) {
|
||||
return lookup.clone();
|
||||
}
|
||||
self.waveforms.insert(url.to_owned(), WaveformLookup::Loading);
|
||||
let (http, limiter, tx, ctx, url) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone(), url.to_owned());
|
||||
self.rt.spawn(async move {
|
||||
let waveform = match limiter.acquire_owned().await {
|
||||
Ok(_permit) => match download(&http, &url).await {
|
||||
Ok(bytes) => tokio::task::spawn_blocking(move || decode_waveform(bytes)).await.ok().flatten(),
|
||||
Err(err) => {
|
||||
warn!("Waveform download failed for {url}: {err}");
|
||||
None
|
||||
},
|
||||
},
|
||||
Err(_) => None,
|
||||
};
|
||||
let _ = tx.send(Loaded::Waveform { url, waveform });
|
||||
ctx.request_repaint();
|
||||
});
|
||||
WaveformLookup::Loading
|
||||
}
|
||||
|
||||
/// Starts downloads that finished and notices clips that ended. Call once per frame. Returns an
|
||||
/// error message when a clip couldn't play.
|
||||
pub fn poll(&mut self) -> Option<String> {
|
||||
let mut error = None;
|
||||
while let Ok(loaded) = self.rx.try_recv() {
|
||||
match loaded {
|
||||
Loaded::Waveform { url, waveform } => {
|
||||
self.waveforms.insert(url, waveform.map_or(WaveformLookup::Failed, |w| WaveformLookup::Ready(Arc::new(w))));
|
||||
},
|
||||
Loaded::Clip { key, bytes } => {
|
||||
// Superseded by another clip (or stopped) while downloading.
|
||||
let Some(clip) = self.current.as_ref().filter(|c| c.key == key) else {
|
||||
continue;
|
||||
};
|
||||
let (volume, start_at) = (if clip.track { self.track_volume } else { PREVIEW_VOLUME }, clip.start_at);
|
||||
if let Err(err) = bytes.and_then(|b| self.play_bytes(b, volume, start_at)) {
|
||||
warn!("Audio playback failed: {err}");
|
||||
self.current = None;
|
||||
error = Some(format!("Couldn't play the audio: {err}"));
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
if self.sink.as_ref().is_some_and(Sink::empty) {
|
||||
self.sink = None;
|
||||
self.current = None;
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
fn start(&mut self, ctx: &egui::Context, key: &str, url: &str, track: bool, start_at: f32) {
|
||||
self.stop();
|
||||
self.current = Some(Clip { key: key.to_owned(), track, start_at });
|
||||
let (http, tx, ctx, key, url) = (self.http.clone(), self.tx.clone(), ctx.clone(), key.to_owned(), url.to_owned());
|
||||
self.rt.spawn(async move {
|
||||
let bytes = download(&http, &url).await;
|
||||
let _ = tx.send(Loaded::Clip { key, bytes });
|
||||
ctx.request_repaint();
|
||||
});
|
||||
}
|
||||
|
||||
fn play_bytes(&mut self, bytes: Arc<[u8]>, volume: f32, start_at: f32) -> Result<(), String> {
|
||||
if self.output.is_none() {
|
||||
self.output = Some(OutputStream::try_default().map_err(|e| e.to_string())?);
|
||||
}
|
||||
let Some((_, handle)) = &self.output else {
|
||||
return Err("no audio output".to_owned());
|
||||
};
|
||||
let sink = Sink::try_new(handle).map_err(|e| e.to_string())?;
|
||||
let source = Decoder::new(Cursor::new(bytes)).map_err(|e| e.to_string())?;
|
||||
sink.set_volume(volume);
|
||||
sink.append(source);
|
||||
if start_at > 0.0 {
|
||||
if let Err(err) = sink.try_seek(Duration::from_secs_f32(start_at)) {
|
||||
warn!("Audio seek failed: {err}");
|
||||
}
|
||||
}
|
||||
self.sink = Some(sink);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn download(http: &reqwest::Client, url: &str) -> Result<Arc<[u8]>, String> {
|
||||
let response = http.get(url).send().await.and_then(|r| r.error_for_status()).map_err(|e| e.to_string())?;
|
||||
response.bytes().await.map(|b| Arc::from(b.as_ref())).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Decodes a whole file into its waveform: the loudest sample of each window, folded down to
|
||||
/// [`PEAK_COUNT`] peaks and scaled so the loudest is 1.
|
||||
fn decode_waveform(bytes: Arc<[u8]>) -> Option<Waveform> {
|
||||
let decoder = match Decoder::new(Cursor::new(bytes)) {
|
||||
Ok(d) => d,
|
||||
Err(err) => {
|
||||
warn!("Waveform decode failed: {err}");
|
||||
return None;
|
||||
},
|
||||
};
|
||||
let (channels, rate) = (usize::from(decoder.channels().max(1)), decoder.sample_rate().max(1));
|
||||
let window = WINDOW_FRAMES * channels;
|
||||
let (mut windows, mut loudest, mut count) = (Vec::new(), 0_u16, 0_usize);
|
||||
for sample in decoder {
|
||||
loudest = loudest.max(sample.unsigned_abs());
|
||||
count += 1;
|
||||
if count % window == 0 {
|
||||
windows.push(f32::from(loudest));
|
||||
loudest = 0;
|
||||
}
|
||||
}
|
||||
if count % window != 0 {
|
||||
windows.push(f32::from(loudest));
|
||||
}
|
||||
if count == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Waveform { peaks: normalized(&resample(&windows, PEAK_COUNT)), duration: (count / channels) as f32 / rate as f32 })
|
||||
}
|
||||
|
||||
/// `values` folded (by maximum) or stretched to exactly `count` entries.
|
||||
pub fn resample(values: &[f32], count: usize) -> Vec<f32> {
|
||||
if values.is_empty() || count == 0 {
|
||||
return vec![0.0; count];
|
||||
}
|
||||
(0..count)
|
||||
.map(|i| {
|
||||
let start = i * values.len() / count;
|
||||
let end = ((i + 1) * values.len() / count).max(start + 1).min(values.len());
|
||||
values[start..end].iter().copied().fold(0.0, f32::max)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalized(values: &[f32]) -> Vec<f32> {
|
||||
let max = values.iter().copied().fold(0.0, f32::max);
|
||||
if max <= 0.0 {
|
||||
return values.to_vec();
|
||||
}
|
||||
values.iter().map(|v| v / max).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resampling_keeps_the_loudest_of_each_span() {
|
||||
assert_eq!(resample(&[0.1, 0.9, 0.2, 0.4], 2), [0.9, 0.4]);
|
||||
assert_eq!(resample(&[0.5, 1.0], 4), [0.5, 0.5, 1.0, 1.0], "short inputs stretch");
|
||||
assert_eq!(resample(&[], 3), [0.0, 0.0, 0.0]);
|
||||
assert_eq!(normalized(&[0.0, 2.0, 1.0]), [0.0, 1.0, 0.5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_a_wav_into_peaks_and_duration() {
|
||||
// One second of 8 kHz mono: silence, then a loud half.
|
||||
let rate = 8000_u32;
|
||||
let samples: Vec<i16> = (0..rate).map(|i| if i < rate / 2 { 0 } else { 16000 }).collect();
|
||||
let mut wav = Vec::new();
|
||||
wav.extend_from_slice(b"RIFF");
|
||||
wav.extend_from_slice(&(36 + samples.len() as u32 * 2).to_le_bytes());
|
||||
wav.extend_from_slice(b"WAVEfmt ");
|
||||
wav.extend_from_slice(&16_u32.to_le_bytes());
|
||||
wav.extend_from_slice(&1_u16.to_le_bytes());
|
||||
wav.extend_from_slice(&1_u16.to_le_bytes());
|
||||
wav.extend_from_slice(&rate.to_le_bytes());
|
||||
wav.extend_from_slice(&(rate * 2).to_le_bytes());
|
||||
wav.extend_from_slice(&2_u16.to_le_bytes());
|
||||
wav.extend_from_slice(&16_u16.to_le_bytes());
|
||||
wav.extend_from_slice(b"data");
|
||||
wav.extend_from_slice(&(samples.len() as u32 * 2).to_le_bytes());
|
||||
for s in &samples {
|
||||
wav.extend_from_slice(&s.to_le_bytes());
|
||||
}
|
||||
let waveform = decode_waveform(Arc::from(wav)).expect("decodes");
|
||||
assert!((waveform.duration - 1.0).abs() < 0.01, "duration {}", waveform.duration);
|
||||
assert_eq!(waveform.peaks.len(), PEAK_COUNT);
|
||||
assert_eq!(waveform.peaks[0], 0.0, "starts silent");
|
||||
assert_eq!(*waveform.peaks.last().unwrap(), 1.0, "ends loud");
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,18 @@
|
||||
//! Images shown in the UI: CDN thumbnails and local previews, as GPU textures.
|
||||
//! Images shown in the UI: CDN thumbnails, animated video previews and local previews, as GPU
|
||||
//! textures.
|
||||
//!
|
||||
//! Downloads run on the tokio runtime (a few at a time) and decode off the UI thread; the UI
|
||||
//! uploads finished images in [`MediaCache::poll`]. Textures nobody has drawn for a while are
|
||||
//! dropped once the cache grows past its budget, so long feeds don't hold every image forever.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use egui::{ColorImage, TextureHandle, TextureOptions};
|
||||
use image::AnimationDecoder;
|
||||
use log::warn;
|
||||
use tokio::runtime::Handle;
|
||||
use tokio::sync::Semaphore;
|
||||
@@ -17,10 +21,14 @@ use tokio::sync::Semaphore;
|
||||
const MAX_CONCURRENT_DOWNLOADS: usize = 6;
|
||||
/// Longest side of a decoded image; larger ones are downscaled before upload.
|
||||
const MAX_DECODED_SIDE: u32 = 2048;
|
||||
/// Animated previews: longest side and frame budget (each frame is its own texture).
|
||||
const ANIMATED_SIDE: u32 = 320;
|
||||
const ANIMATED_MAX_FRAMES: usize = 48;
|
||||
/// Textures kept even when unused (roughly two screens of thumbnails).
|
||||
const SOFT_CAPACITY: usize = 160;
|
||||
/// Frames an image may go undrawn before it can be evicted.
|
||||
/// Frames an image may go undrawn before it can be evicted (animations go sooner: they're heavy).
|
||||
const EVICT_AFTER_FRAMES: u64 = 600;
|
||||
const EVICT_ANIMATED_AFTER_FRAMES: u64 = 120;
|
||||
|
||||
pub struct MediaCache {
|
||||
rt: Handle,
|
||||
@@ -40,14 +48,30 @@ pub enum Lookup {
|
||||
}
|
||||
|
||||
enum Entry {
|
||||
Loading { last_used: u64 },
|
||||
Ready { texture: TextureHandle, last_used: u64 },
|
||||
Loading {
|
||||
last_used: u64,
|
||||
},
|
||||
Ready {
|
||||
texture: TextureHandle,
|
||||
last_used: u64,
|
||||
},
|
||||
/// Frames with how long each shows (seconds), and their total.
|
||||
Animated {
|
||||
frames: Vec<(TextureHandle, f32)>,
|
||||
total: f32,
|
||||
last_used: u64,
|
||||
},
|
||||
Failed,
|
||||
}
|
||||
|
||||
struct Loaded {
|
||||
key: String,
|
||||
image: Option<ColorImage>,
|
||||
decoded: Option<Decoded>,
|
||||
}
|
||||
|
||||
enum Decoded {
|
||||
Still(ColorImage),
|
||||
Frames(Vec<(ColorImage, f32)>),
|
||||
}
|
||||
|
||||
impl MediaCache {
|
||||
@@ -58,23 +82,13 @@ impl MediaCache {
|
||||
|
||||
/// The texture for `url`, starting its download the first time it's asked for.
|
||||
pub fn get(&mut self, ctx: &egui::Context, url: &str) -> Lookup {
|
||||
let frame = self.frame;
|
||||
match self.entries.get_mut(url) {
|
||||
Some(Entry::Ready { texture, last_used }) => {
|
||||
*last_used = frame;
|
||||
Lookup::Ready(texture.clone())
|
||||
},
|
||||
Some(Entry::Loading { last_used }) => {
|
||||
*last_used = frame;
|
||||
Lookup::Loading
|
||||
},
|
||||
Some(Entry::Failed) => Lookup::Failed,
|
||||
None => {
|
||||
self.entries.insert(url.to_owned(), Entry::Loading { last_used: frame });
|
||||
self.spawn_download(ctx, url.to_owned());
|
||||
Lookup::Loading
|
||||
},
|
||||
}
|
||||
self.lookup(ctx, url, url, false)
|
||||
}
|
||||
|
||||
/// The current frame of an animated image (a video's animated preview); still images come
|
||||
/// back as they are.
|
||||
pub fn get_animated(&mut self, ctx: &egui::Context, url: &str) -> Lookup {
|
||||
self.lookup(ctx, &format!("anim:{url}"), url, true)
|
||||
}
|
||||
|
||||
/// Decodes `bytes` off the UI thread and registers the result under `key`.
|
||||
@@ -85,8 +99,8 @@ impl MediaCache {
|
||||
self.entries.insert(key.to_owned(), Entry::Loading { last_used: self.frame });
|
||||
let (tx, ctx, key) = (self.tx.clone(), ctx.clone(), key.to_owned());
|
||||
self.rt.spawn_blocking(move || {
|
||||
let image = decode(&bytes);
|
||||
let _ = tx.send(Loaded { key, image });
|
||||
let decoded = decode(&bytes).map(Decoded::Still);
|
||||
let _ = tx.send(Loaded { key, decoded });
|
||||
ctx.request_repaint();
|
||||
});
|
||||
}
|
||||
@@ -94,39 +108,78 @@ impl MediaCache {
|
||||
/// Uploads finished downloads and evicts stale textures. Call once per frame.
|
||||
pub fn poll(&mut self, ctx: &egui::Context) {
|
||||
self.frame += 1;
|
||||
while let Ok(Loaded { key, image }) = self.rx.try_recv() {
|
||||
while let Ok(Loaded { key, decoded }) = self.rx.try_recv() {
|
||||
// Dropped while loading (evicted): don't resurrect it.
|
||||
let Some(entry) = self.entries.get_mut(&key) else {
|
||||
continue;
|
||||
};
|
||||
*entry = match image {
|
||||
Some(image) => Entry::Ready { texture: ctx.load_texture(&key, image, TextureOptions::LINEAR), last_used: self.frame },
|
||||
*entry = match decoded {
|
||||
Some(Decoded::Still(image)) => Entry::Ready { texture: ctx.load_texture(&key, image, TextureOptions::LINEAR), last_used: self.frame },
|
||||
Some(Decoded::Frames(frames)) => {
|
||||
let total = frames.iter().map(|(_, d)| d).sum();
|
||||
let frames = frames.into_iter().enumerate().map(|(i, (img, d))| (ctx.load_texture(format!("{key}#{i}"), img, TextureOptions::LINEAR), d)).collect();
|
||||
Entry::Animated { frames, total, last_used: self.frame }
|
||||
},
|
||||
None => Entry::Failed,
|
||||
};
|
||||
}
|
||||
if self.entries.len() > SOFT_CAPACITY {
|
||||
let frame = self.frame;
|
||||
self.entries.retain(|_, e| match e {
|
||||
Entry::Ready { last_used, .. } | Entry::Loading { last_used } => frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES,
|
||||
Entry::Failed => true,
|
||||
});
|
||||
let frame = self.frame;
|
||||
let crowded = self.entries.len() > SOFT_CAPACITY;
|
||||
self.entries.retain(|_, e| match e {
|
||||
Entry::Animated { last_used, .. } => frame.saturating_sub(*last_used) < EVICT_ANIMATED_AFTER_FRAMES,
|
||||
Entry::Ready { last_used, .. } | Entry::Loading { last_used } => !crowded || frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES,
|
||||
Entry::Failed => true,
|
||||
});
|
||||
}
|
||||
|
||||
fn lookup(&mut self, ctx: &egui::Context, key: &str, url: &str, animated: bool) -> Lookup {
|
||||
let frame = self.frame;
|
||||
match self.entries.get_mut(key) {
|
||||
Some(Entry::Ready { texture, last_used }) => {
|
||||
*last_used = frame;
|
||||
Lookup::Ready(texture.clone())
|
||||
},
|
||||
Some(Entry::Animated { frames, total, last_used }) => {
|
||||
*last_used = frame;
|
||||
let t = (ctx.input(|i| i.time) as f32).rem_euclid(total.max(0.01));
|
||||
let (mut acc, mut index) = (0.0, frames.len() - 1);
|
||||
for (i, (_, d)) in frames.iter().enumerate() {
|
||||
acc += d;
|
||||
if t < acc {
|
||||
index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ctx.request_repaint_after(Duration::from_secs_f32((acc - t).max(0.01)));
|
||||
Lookup::Ready(frames[index].0.clone())
|
||||
},
|
||||
Some(Entry::Loading { last_used }) => {
|
||||
*last_used = frame;
|
||||
Lookup::Loading
|
||||
},
|
||||
Some(Entry::Failed) => Lookup::Failed,
|
||||
None => {
|
||||
self.entries.insert(key.to_owned(), Entry::Loading { last_used: frame });
|
||||
self.spawn_download(ctx, key.to_owned(), url.to_owned(), animated);
|
||||
Lookup::Loading
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_download(&self, ctx: &egui::Context, url: String) {
|
||||
fn spawn_download(&self, ctx: &egui::Context, key: String, url: String, animated: bool) {
|
||||
let (http, limiter, tx, ctx) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone());
|
||||
self.rt.spawn(async move {
|
||||
let image = match limiter.acquire_owned().await {
|
||||
Ok(_permit) => download(&http, &url).await,
|
||||
let decoded = match limiter.acquire_owned().await {
|
||||
Ok(_permit) => download(&http, &url, animated).await,
|
||||
Err(_) => None,
|
||||
};
|
||||
let _ = tx.send(Loaded { key: url, image });
|
||||
let _ = tx.send(Loaded { key, decoded });
|
||||
ctx.request_repaint();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn download(http: &reqwest::Client, url: &str) -> Option<ColorImage> {
|
||||
async fn download(http: &reqwest::Client, url: &str, animated: bool) -> Option<Decoded> {
|
||||
let response = match http.get(url).send().await.and_then(|r| r.error_for_status()) {
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
@@ -135,7 +188,7 @@ async fn download(http: &reqwest::Client, url: &str) -> Option<ColorImage> {
|
||||
},
|
||||
};
|
||||
let bytes = response.bytes().await.ok()?;
|
||||
tokio::task::spawn_blocking(move || decode(&bytes)).await.ok().flatten()
|
||||
tokio::task::spawn_blocking(move || if animated { decode_frames(&bytes).map(Decoded::Frames).or_else(|| decode(&bytes).map(Decoded::Still)) } else { decode(&bytes).map(Decoded::Still) }).await.ok().flatten()
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8]) -> Option<ColorImage> {
|
||||
@@ -147,7 +200,45 @@ fn decode(bytes: &[u8]) -> Option<ColorImage> {
|
||||
},
|
||||
};
|
||||
let image = if image.width().max(image.height()) > MAX_DECODED_SIDE { image.thumbnail(MAX_DECODED_SIDE, MAX_DECODED_SIDE) } else { image };
|
||||
let rgba = image.to_rgba8();
|
||||
Some(to_color_image(image.to_rgba8()))
|
||||
}
|
||||
|
||||
/// The frames of an animated GIF or WebP (subsampled to the budget), or `None` when it isn't
|
||||
/// animated.
|
||||
fn decode_frames(bytes: &[u8]) -> Option<Vec<(ColorImage, f32)>> {
|
||||
let frames = match image::guess_format(bytes).ok()? {
|
||||
image::ImageFormat::Gif => image::codecs::gif::GifDecoder::new(Cursor::new(bytes)).ok()?.into_frames().collect_frames().ok()?,
|
||||
image::ImageFormat::WebP => {
|
||||
let decoder = image::codecs::webp::WebPDecoder::new(Cursor::new(bytes)).ok()?;
|
||||
if !decoder.has_animation() {
|
||||
return None;
|
||||
}
|
||||
decoder.into_frames().collect_frames().ok()?
|
||||
},
|
||||
_ => return None,
|
||||
};
|
||||
if frames.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
// Keep every n-th frame, folding the skipped frames' time into the kept one.
|
||||
let step = frames.len().div_ceil(ANIMATED_MAX_FRAMES);
|
||||
let mut out = Vec::with_capacity(frames.len() / step + 1);
|
||||
for chunk in frames.chunks(step) {
|
||||
let secs: f32 = chunk
|
||||
.iter()
|
||||
.map(|f| {
|
||||
let (n, d) = f.delay().numer_denom_ms();
|
||||
n as f32 / d.max(1) as f32 / 1000.0
|
||||
})
|
||||
.sum();
|
||||
let image = image::DynamicImage::ImageRgba8(chunk[0].buffer().clone()).thumbnail(ANIMATED_SIDE, ANIMATED_SIDE);
|
||||
// Some encoders write 0 ms delays; browsers treat those as ~100 ms.
|
||||
out.push((to_color_image(image.to_rgba8()), if secs < 0.02 { 0.1 * chunk.len() as f32 } else { secs }));
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn to_color_image(rgba: image::RgbaImage) -> ColorImage {
|
||||
let size = [rgba.width() as usize, rgba.height() as usize];
|
||||
Some(ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()))
|
||||
ColorImage::from_rgba_unmultiplied(size, rgba.as_raw())
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
//! Everything that talks to the ArtCraft API. Calls run on a tokio runtime and report back as
|
||||
//! [`Event`]s, which the UI drains once per frame; nothing here blocks the UI thread.
|
||||
|
||||
pub mod audio;
|
||||
pub mod media_cache;
|
||||
pub mod session;
|
||||
pub mod video;
|
||||
pub mod wire;
|
||||
|
||||
use std::future::Future;
|
||||
@@ -36,7 +38,7 @@ use tokens::tokens::media_files::MediaFileToken;
|
||||
use crate::feed::types::MediaKind;
|
||||
use crate::models::{self, ModelInfo};
|
||||
use session::Credentials;
|
||||
use wire::{BatchMedia, MediaFile, PromptResponse, SessionInfo, SessionJobs, SessionUser, UploadResponse, UserMediaList};
|
||||
use wire::{AudioModels, BatchMedia, CharactersPage, MediaFile, PromptResponse, SessionInfo, SessionJobs, SessionUser, UploadResponse, UserMediaList};
|
||||
|
||||
/// Items per library page (`useGalleryData`'s `PAGE_SIZE`).
|
||||
pub const LIBRARY_PAGE_SIZE: u32 = 40;
|
||||
@@ -45,6 +47,8 @@ const MODEL_LIST_ATTEMPTS: u32 = 3;
|
||||
const MODEL_LIST_BACKOFF: Duration = Duration::from_millis(250);
|
||||
/// The share link base (`SHARE_URL_BASE`).
|
||||
pub const SHARE_URL_BASE: &str = "https://getartcraft.com/media/";
|
||||
/// Characters are made for this model (the Tauri app's `CharactersModal`).
|
||||
const CHARACTER_MODEL: &str = "seedance_2p0";
|
||||
/// Sent on every request (`artcraft_client_identity::ARTCRAFT_DESKTOP_USER_AGENT`).
|
||||
const USER_AGENT: &str = "storyteller-client/1.0";
|
||||
|
||||
@@ -53,6 +57,7 @@ const USER_AGENT: &str = "storyteller-client/1.0";
|
||||
pub enum Modality {
|
||||
Image,
|
||||
Video,
|
||||
Audio,
|
||||
}
|
||||
|
||||
impl Modality {
|
||||
@@ -60,6 +65,7 @@ impl Modality {
|
||||
match self {
|
||||
Modality::Image => "image",
|
||||
Modality::Video => "video",
|
||||
Modality::Audio => "audio",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,6 +75,7 @@ impl From<MediaKind> for Modality {
|
||||
match kind {
|
||||
MediaKind::Image => Modality::Image,
|
||||
MediaKind::Video => Modality::Video,
|
||||
MediaKind::Audio => Modality::Audio,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,6 +105,8 @@ pub enum Event {
|
||||
EnqueueFailed {
|
||||
modality: Modality,
|
||||
message: String,
|
||||
/// The HTTP status, when the server answered.
|
||||
status: Option<u16>,
|
||||
},
|
||||
Uploaded {
|
||||
ref_id: u64,
|
||||
@@ -119,6 +128,14 @@ pub enum Event {
|
||||
items: Vec<MediaFile>,
|
||||
},
|
||||
Prompt(wire::Prompt),
|
||||
/// The user's characters, newest first.
|
||||
Characters(Vec<wire::Character>),
|
||||
/// A character is being made by job `job_token`.
|
||||
CharacterCreating {
|
||||
job_token: String,
|
||||
name: String,
|
||||
},
|
||||
CharacterChanged,
|
||||
Deleted(String),
|
||||
Saved(PathBuf),
|
||||
Toast {
|
||||
@@ -322,7 +339,8 @@ impl Backend {
|
||||
|
||||
/// Fetches the OmniGen model listing for `modality`.
|
||||
pub fn load_models(&self, modality: Modality) {
|
||||
let host = self.host.clone();
|
||||
let api = self.api();
|
||||
let host = api.host.clone();
|
||||
self.spawn(move |tx| async move {
|
||||
let mut last_err = String::new();
|
||||
for attempt in 0..MODEL_LIST_ATTEMPTS {
|
||||
@@ -330,15 +348,16 @@ impl Backend {
|
||||
tokio::time::sleep(MODEL_LIST_BACKOFF).await;
|
||||
}
|
||||
let result = match modality {
|
||||
Modality::Image => omni_gen_list_image_models(OmniGenListImageModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::image_model).collect::<Vec<_>>()),
|
||||
Modality::Video => omni_gen_list_video_models(OmniGenListVideoModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::video_model).collect::<Vec<_>>()),
|
||||
Modality::Image => omni_gen_list_image_models(OmniGenListImageModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::image_model).collect::<Vec<_>>()).map_err(|e| api_message(&e)),
|
||||
Modality::Video => omni_gen_list_video_models(OmniGenListVideoModelsArgs { api_host: &host, maybe_creds: None, provider: None }).await.map(|r| r.models.iter().map(models::video_model).collect::<Vec<_>>()).map_err(|e| api_message(&e)),
|
||||
Modality::Audio => api.get_json::<AudioModels>("/v1/omni_gen/models/audio").await.map(|r| r.models.iter().map(models::audio_model).collect::<Vec<_>>()),
|
||||
};
|
||||
match result {
|
||||
Ok(models) => {
|
||||
let _ = tx.send(Event::Models(modality, Ok(models)));
|
||||
return;
|
||||
},
|
||||
Err(err) => last_err = api_message(&err),
|
||||
Err(err) => last_err = err,
|
||||
}
|
||||
}
|
||||
error!("Listing {modality:?} models failed: {last_err}");
|
||||
@@ -367,7 +386,7 @@ impl Backend {
|
||||
let api = self.api();
|
||||
self.spawn(move |tx| async move {
|
||||
let Some(creds) = api.creds.as_ref() else {
|
||||
let _ = tx.send(Event::EnqueueFailed { modality, message: "Please sign in to generate.".to_owned() });
|
||||
let _ = tx.send(Event::EnqueueFailed { modality, message: "Please sign in to generate.".to_owned(), status: None });
|
||||
return;
|
||||
};
|
||||
fields.entry("idempotency_token").or_insert_with(|| Value::String(uuid::Uuid::new_v4().to_string()));
|
||||
@@ -376,14 +395,14 @@ impl Backend {
|
||||
Ok(response) => {
|
||||
let tokens = job_tokens(&response);
|
||||
if tokens.is_empty() {
|
||||
let _ = tx.send(Event::EnqueueFailed { modality, message: "The server didn't start a job.".to_owned() });
|
||||
let _ = tx.send(Event::EnqueueFailed { modality, message: "The server didn't start a job.".to_owned(), status: None });
|
||||
} else {
|
||||
let _ = tx.send(Event::Enqueued { modality, job_tokens: tokens, meta });
|
||||
}
|
||||
api.send_credits(&tx).await;
|
||||
},
|
||||
Err(err) => {
|
||||
let _ = tx.send(Event::EnqueueFailed { modality, message: api_message(&err) });
|
||||
let _ = tx.send(Event::EnqueueFailed { modality, message: api_message(&err), status: api_status(&err) });
|
||||
},
|
||||
}
|
||||
});
|
||||
@@ -485,6 +504,92 @@ impl Backend {
|
||||
});
|
||||
}
|
||||
|
||||
/// Loads every page of the user's characters.
|
||||
pub fn load_characters(&self) {
|
||||
let api = self.api();
|
||||
if api.creds.is_none() {
|
||||
return;
|
||||
}
|
||||
self.spawn(move |tx| async move {
|
||||
let mut all = Vec::new();
|
||||
let mut cursor: Option<i64> = None;
|
||||
loop {
|
||||
let path = match cursor {
|
||||
Some(c) => format!("/v1/characters/session?cursor={c}"),
|
||||
None => "/v1/characters/session".to_owned(),
|
||||
};
|
||||
match api.request_json::<CharactersPage>(reqwest::Method::GET, &path, None).await {
|
||||
Ok(page) => {
|
||||
all.extend(page.characters);
|
||||
match page.next_cursor {
|
||||
Some(next) if Some(next) != cursor => cursor = Some(next),
|
||||
_ => break,
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
warn!("Loading characters failed: {err}");
|
||||
break;
|
||||
},
|
||||
}
|
||||
}
|
||||
let _ = tx.send(Event::Characters(all));
|
||||
});
|
||||
}
|
||||
|
||||
/// Starts making a character from an uploaded reference image (`POST /v1/character/create`).
|
||||
pub fn create_character(&self, image_token: String, name: String, description: Option<String>) {
|
||||
let api = self.api();
|
||||
self.spawn(move |tx| async move {
|
||||
let body = serde_json::json!({ "image_media_token": image_token, "model": CHARACTER_MODEL, "uuid_idempotency_token": uuid::Uuid::new_v4().to_string(), "character_name": name, "character_description": description });
|
||||
match api.request_json::<Value>(reqwest::Method::POST, "/v1/character/create", Some(body)).await {
|
||||
Ok(r) => match r.get("inference_job_token").and_then(Value::as_str) {
|
||||
Some(job) => {
|
||||
let _ = tx.send(Event::CharacterCreating { job_token: job.to_owned(), name });
|
||||
},
|
||||
None => {
|
||||
let _ = tx.send(Event::Toast { error: true, message: "Failed to create character".to_owned() });
|
||||
},
|
||||
},
|
||||
Err(err) => {
|
||||
let _ = tx.send(Event::Toast { error: true, message: if err.is_empty() { "Failed to create character".to_owned() } else { err } });
|
||||
},
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Renames or re-describes a character (`POST /v1/character/edit`).
|
||||
pub fn edit_character(&self, token: String, name: String, description: String) {
|
||||
let api = self.api();
|
||||
self.spawn(move |tx| async move {
|
||||
let body = serde_json::json!({ "token": token, "updated_name": name, "updated_description": description });
|
||||
match api.request_json::<Value>(reqwest::Method::POST, "/v1/character/edit", Some(body)).await {
|
||||
Ok(_) => {
|
||||
let _ = tx.send(Event::Toast { error: false, message: "Character updated".to_owned() });
|
||||
let _ = tx.send(Event::CharacterChanged);
|
||||
},
|
||||
Err(err) => {
|
||||
let _ = tx.send(Event::Toast { error: true, message: format!("Failed to update character: {err}") });
|
||||
},
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// `DELETE /v1/character/{token}`.
|
||||
pub fn delete_character(&self, token: String, name: String) {
|
||||
let api = self.api();
|
||||
self.spawn(move |tx| async move {
|
||||
match api.request_json::<Value>(reqwest::Method::DELETE, &format!("/v1/character/{token}"), None).await {
|
||||
Ok(_) => {
|
||||
let _ = tx.send(Event::Toast { error: false, message: format!("Character \"{name}\" deleted") });
|
||||
let _ = tx.send(Event::CharacterChanged);
|
||||
},
|
||||
Err(err) => {
|
||||
let _ = tx.send(Event::Toast { error: true, message: format!("Failed to delete character: {err}") });
|
||||
},
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub fn delete_media(&self, token: String) {
|
||||
let api = self.api();
|
||||
self.spawn(move |tx| async move {
|
||||
@@ -589,8 +694,16 @@ struct Api {
|
||||
impl Api {
|
||||
/// A GET with the desktop app's identity headers and cookies, decoded leniently.
|
||||
async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, String> {
|
||||
self.request_json(reqwest::Method::GET, path, None).await
|
||||
}
|
||||
|
||||
/// Any JSON call with the desktop app's identity headers and cookies, decoded leniently.
|
||||
async fn request_json<T: DeserializeOwned>(&self, method: reqwest::Method, path: &str, body: Option<Value>) -> Result<T, String> {
|
||||
let url = format!("{}{path}", self.host.to_api_hostname_and_scheme());
|
||||
let mut request = self.http.get(&url).header("Accept", "application/json").header("Origin", self.host.request_origin());
|
||||
let mut request = self.http.request(method, &url).header("Accept", "application/json").header("Origin", self.host.request_origin());
|
||||
if let Some(body) = body {
|
||||
request = request.json(&body);
|
||||
}
|
||||
if let Some(cookie) = self.creds.as_ref().and_then(StorytellerCredentialSet::maybe_as_cookie_header) {
|
||||
request = request.header("Cookie", cookie);
|
||||
}
|
||||
@@ -644,7 +757,7 @@ impl Api {
|
||||
}
|
||||
let info = self.get_json::<FileResponse>(&format!("/v1/media_files/file/{token}")).await.ok();
|
||||
let file = info.map(|r| r.media_file);
|
||||
UploadedMedia { duration_secs: file.as_ref().and_then(|f| f.maybe_duration_millis).map_or(0.0, |ms| ms as f32 / 1000.0), thumbnail: file.as_ref().and_then(|f| f.media_links.thumbnail(256)), full_url: file.as_ref().map(|f| f.media_links.cdn_url.clone()).filter(|u| !u.is_empty()), token }
|
||||
UploadedMedia { duration_secs: file.as_ref().and_then(|f| f.maybe_duration_millis).map_or(0.0, |ms| ms as f32 / 1000.0), thumbnail: file.as_ref().and_then(|f| f.thumbnail(256)), full_url: file.as_ref().map(|f| f.media_links.cdn_url.clone()).filter(|u| !u.is_empty()), token }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -687,6 +800,24 @@ pub fn api_message(err: &StorytellerError) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// The HTTP status behind a client error, when there was a response.
|
||||
fn api_status(err: &StorytellerError) -> Option<u16> {
|
||||
let StorytellerError::Api(api) = err else {
|
||||
return None;
|
||||
};
|
||||
Some(match api {
|
||||
ApiError::InvalidRequest(_) => 400,
|
||||
ApiError::Unauthorized(_) => 401,
|
||||
ApiError::PaymentRequired(_) => 402,
|
||||
ApiError::Forbidden(_) => 403,
|
||||
ApiError::NotFound(_) => 404,
|
||||
ApiError::TooManyRequests(_) => 429,
|
||||
ApiError::InternalServerError { .. } => 500,
|
||||
ApiError::UncategorizedBadResponseWithStatusAndBody { status_code, .. } => status_code.as_u16(),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn api_error_message(err: &ApiError) -> String {
|
||||
let body = match err {
|
||||
ApiError::InvalidRequest(b) | ApiError::Unauthorized(b) | ApiError::PaymentRequired(b) | ApiError::Forbidden(b) | ApiError::NotFound(b) | ApiError::TooManyRequests(b) => Some(b.as_str()),
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
//! In-app video playback through FFmpeg (when it's installed): `ffprobe` reads the clip's size,
|
||||
//! rate and length, one `ffmpeg` streams RGBA frames into a texture and another streams PCM into
|
||||
//! rodio. Pausing and seeking restart both at the new position. Without FFmpeg the lightbox opens
|
||||
//! videos in the system player instead.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::io::Read;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, channel, sync_channel};
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use egui::{Align2, Color32, ColorImage, FontId, Rect, Sense, TextureHandle, TextureOptions, Ui, pos2, vec2};
|
||||
use log::warn;
|
||||
use rodio::{OutputStream, OutputStreamHandle, Sink, Source};
|
||||
|
||||
use crate::theme;
|
||||
use crate::ui::icons::{self, Icon};
|
||||
|
||||
/// Frames are scaled down to at most this width (the lightbox never shows them larger).
|
||||
const MAX_WIDTH: u32 = 1280;
|
||||
const AUDIO_RATE: u32 = 44_100;
|
||||
/// Decoded frames buffered ahead of the clock (back-pressure on `ffmpeg`).
|
||||
const FRAME_QUEUE: usize = 4;
|
||||
const CONTROLS_HEIGHT: f32 = 40.0;
|
||||
|
||||
/// Whether `ffmpeg` and `ffprobe` are on the PATH (checked once).
|
||||
pub fn ffmpeg_available() -> bool {
|
||||
static AVAILABLE: OnceLock<bool> = OnceLock::new();
|
||||
*AVAILABLE.get_or_init(|| ["ffmpeg", "ffprobe"].iter().all(|tool| command(tool).arg("-version").stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success())))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct VideoInfo {
|
||||
width: u32,
|
||||
height: u32,
|
||||
fps: f64,
|
||||
duration: f64,
|
||||
}
|
||||
|
||||
pub struct VideoPlayer {
|
||||
url: String,
|
||||
probe: Option<Receiver<Result<VideoInfo, String>>>,
|
||||
info: Option<VideoInfo>,
|
||||
stream: Option<Stream>,
|
||||
texture: Option<TextureHandle>,
|
||||
/// Media time of the frame on screen.
|
||||
position: f64,
|
||||
muted: bool,
|
||||
error: Option<String>,
|
||||
output: Option<(OutputStream, OutputStreamHandle)>,
|
||||
}
|
||||
|
||||
/// One run of the decoders from a start position.
|
||||
struct Stream {
|
||||
start: f64,
|
||||
started: Instant,
|
||||
frames: Receiver<(ColorImage, f64)>,
|
||||
next: Option<(ColorImage, f64)>,
|
||||
ended: bool,
|
||||
children: Vec<Child>,
|
||||
sink: Option<Sink>,
|
||||
}
|
||||
|
||||
impl Drop for Stream {
|
||||
fn drop(&mut self) {
|
||||
for child in &mut self.children {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoPlayer {
|
||||
/// Starts probing `url`; playback begins as soon as its size is known.
|
||||
pub fn new(url: &str) -> Self {
|
||||
let (tx, rx) = channel();
|
||||
let probe_url = url.to_owned();
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send(probe(&probe_url));
|
||||
});
|
||||
Self { url: url.to_owned(), probe: Some(rx), info: None, stream: None, texture: None, position: 0.0, muted: false, error: None, output: None }
|
||||
}
|
||||
|
||||
pub fn url(&self) -> &str {
|
||||
&self.url
|
||||
}
|
||||
|
||||
/// Draws the video fitted inside `rect` with its controls; `poster` (the still frame) shows
|
||||
/// until the first decoded frame arrives.
|
||||
pub fn ui(&mut self, ui: &mut Ui, rect: Rect, poster: Option<&TextureHandle>) {
|
||||
self.poll_probe();
|
||||
let ctx = ui.ctx().clone();
|
||||
self.advance(&ctx);
|
||||
ui.painter().rect_filled(rect, 0.0, Color32::BLACK);
|
||||
let video_area = Rect::from_min_max(rect.min, pos2(rect.right(), rect.bottom() - CONTROLS_HEIGHT));
|
||||
match (&self.texture, &self.error) {
|
||||
(_, Some(err)) => {
|
||||
ui.painter().text(video_area.center(), Align2::CENTER_CENTER, format!("Couldn't play this video: {err}"), FontId::new(13.0, egui::FontFamily::Proportional), theme::BAD);
|
||||
},
|
||||
(Some(texture), None) => {
|
||||
let fit = crate::overlays::lightbox::fit_inside(texture.size_vec2(), video_area.shrink(4.0).size());
|
||||
egui::Image::new(texture).paint_at(ui, Rect::from_center_size(video_area.center(), fit));
|
||||
},
|
||||
(None, None) => {
|
||||
if let Some(poster) = poster {
|
||||
let fit = crate::overlays::lightbox::fit_inside(poster.size_vec2(), video_area.shrink(4.0).size());
|
||||
egui::Image::new(poster).tint(Color32::from_gray(150)).paint_at(ui, Rect::from_center_size(video_area.center(), fit));
|
||||
}
|
||||
icons::paint_spinner(ui, Rect::from_center_size(video_area.center(), vec2(32.0, 32.0)), Color32::WHITE);
|
||||
},
|
||||
}
|
||||
// Clicking the picture toggles play/pause.
|
||||
let click = ui.interact(video_area, ui.id().with(("video-surface", &self.url)), Sense::click());
|
||||
if click.clicked() {
|
||||
self.toggle_pause();
|
||||
}
|
||||
self.controls(ui, Rect::from_min_max(pos2(rect.left(), rect.bottom() - CONTROLS_HEIGHT), rect.max));
|
||||
if self.stream.is_some() {
|
||||
ctx.request_repaint_after(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
fn controls(&mut self, ui: &mut Ui, bar: Rect) {
|
||||
let p = ui.painter().clone();
|
||||
p.rect_filled(bar, 0.0, theme::fade(theme::PANEL, 0.95));
|
||||
let playing = self.stream.is_some();
|
||||
let button = Rect::from_center_size(pos2(bar.left() + 22.0, bar.center().y), vec2(28.0, 28.0));
|
||||
let resp = ui.interact(button, ui.id().with(("video-play", &self.url)), Sense::click()).on_hover_text(if playing { "Pause" } else { "Play" }).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
if playing {
|
||||
for dx in [-3.5, 3.5] {
|
||||
p.rect_filled(Rect::from_center_size(button.center() + vec2(dx, 0.0), vec2(3.0, 12.0)), 0.0, theme::INK);
|
||||
}
|
||||
} else {
|
||||
icons::paint(&p, button.shrink(7.0), Icon::Play, theme::INK);
|
||||
}
|
||||
if resp.clicked() {
|
||||
self.toggle_pause();
|
||||
}
|
||||
let duration = self.info.map_or(0.0, |i| i.duration);
|
||||
let time = format!("{} / {}", clock(self.position), clock(duration));
|
||||
let label = p.text(pos2(button.right() + 8.0, bar.center().y), Align2::LEFT_CENTER, time, FontId::new(11.5, egui::FontFamily::Monospace), theme::MUTED);
|
||||
let mute = Rect::from_center_size(pos2(bar.right() - 22.0, bar.center().y), vec2(28.0, 28.0));
|
||||
let mute_resp = ui.interact(mute, ui.id().with(("video-mute", &self.url)), Sense::click()).on_hover_text(if self.muted { "Unmute" } else { "Mute" }).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
icons::paint(&p, mute.shrink(7.0), if self.muted { Icon::VolumeOff } else { Icon::Volume }, theme::INK);
|
||||
if mute_resp.clicked() {
|
||||
self.muted = !self.muted;
|
||||
if let Some(sink) = self.stream.as_ref().and_then(|s| s.sink.as_ref()) {
|
||||
sink.set_volume(if self.muted { 0.0 } else { 1.0 });
|
||||
}
|
||||
}
|
||||
// The scrubber: click or drag to seek.
|
||||
let track = Rect::from_min_max(pos2(label.right() + 14.0, bar.center().y - 8.0), pos2(mute.left() - 12.0, bar.center().y + 8.0));
|
||||
let scrub = ui.interact(track, ui.id().with(("video-scrub", &self.url)), Sense::click_and_drag()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
let line = Rect::from_center_size(track.center(), vec2(track.width(), 4.0));
|
||||
p.rect_filled(line, egui::CornerRadius::same(2), theme::WASH_HOVER);
|
||||
let fraction = if duration > 0.0 { (self.position / duration).clamp(0.0, 1.0) as f32 } else { 0.0 };
|
||||
p.rect_filled(Rect::from_min_size(line.min, vec2(line.width() * fraction, line.height())), egui::CornerRadius::same(2), theme::ACCENT_400);
|
||||
if (scrub.clicked() || scrub.drag_stopped()) && duration > 0.0 {
|
||||
if let Some(pos) = scrub.interact_pointer_pos() {
|
||||
let t = (((pos.x - line.left()) / line.width()).clamp(0.0, 1.0) as f64) * duration;
|
||||
self.seek(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_probe(&mut self) {
|
||||
let Some(rx) = &self.probe else {
|
||||
return;
|
||||
};
|
||||
match rx.try_recv() {
|
||||
Ok(Ok(info)) => {
|
||||
self.info = Some(info);
|
||||
self.probe = None;
|
||||
self.play_from(0.0);
|
||||
},
|
||||
Ok(Err(err)) => {
|
||||
self.error = Some(err);
|
||||
self.probe = None;
|
||||
},
|
||||
Err(_) => {},
|
||||
}
|
||||
}
|
||||
|
||||
/// Shows the newest frame that's due; loops at the end.
|
||||
fn advance(&mut self, ctx: &egui::Context) {
|
||||
let Some(stream) = &mut self.stream else {
|
||||
return;
|
||||
};
|
||||
let now = stream.start + stream.started.elapsed().as_secs_f64();
|
||||
let mut shown = None;
|
||||
loop {
|
||||
if stream.next.is_none() {
|
||||
match stream.frames.try_recv() {
|
||||
Ok(frame) => stream.next = Some(frame),
|
||||
Err(TryRecvError::Disconnected) => {
|
||||
stream.ended = true;
|
||||
break;
|
||||
},
|
||||
Err(TryRecvError::Empty) => break,
|
||||
}
|
||||
}
|
||||
match &stream.next {
|
||||
Some((_, pts)) if *pts <= now => shown = stream.next.take(),
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
if let Some((image, pts)) = shown {
|
||||
self.position = pts;
|
||||
match &mut self.texture {
|
||||
Some(t) => t.set(image, TextureOptions::LINEAR),
|
||||
None => self.texture = Some(ctx.load_texture(format!("video:{}", self.url), image, TextureOptions::LINEAR)),
|
||||
}
|
||||
}
|
||||
if stream.ended && stream.next.is_none() {
|
||||
self.play_from(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn toggle_pause(&mut self) {
|
||||
if self.stream.take().is_none() && self.info.is_some() {
|
||||
self.play_from(self.position);
|
||||
}
|
||||
}
|
||||
|
||||
fn seek(&mut self, t: f64) {
|
||||
self.position = t;
|
||||
self.play_from(t);
|
||||
}
|
||||
|
||||
fn play_from(&mut self, start: f64) {
|
||||
self.stream = None;
|
||||
let Some(info) = self.info else {
|
||||
return;
|
||||
};
|
||||
match self.spawn(info, start) {
|
||||
Ok(stream) => self.stream = Some(stream),
|
||||
Err(err) => self.error = Some(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn(&mut self, info: VideoInfo, start: f64) -> Result<Stream, String> {
|
||||
let (w, h) = output_size(info);
|
||||
let mut video = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{start:.3}"), "-i", &self.url, "-an", "-vf", &format!("scale={w}:{h}"), "-f", "rawvideo", "-pix_fmt", "rgba", "-"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(|e| format!("ffmpeg: {e}"))?;
|
||||
let stdout = video.stdout.take().ok_or("ffmpeg has no output")?;
|
||||
let (frame_tx, frame_rx) = sync_channel(FRAME_QUEUE);
|
||||
let fps = info.fps.max(1.0);
|
||||
std::thread::spawn(move || read_frames(stdout, w, h, start, fps, frame_tx));
|
||||
let mut children = vec![video];
|
||||
let sink = self.spawn_audio(start, &mut children);
|
||||
Ok(Stream { start, started: Instant::now(), frames: frame_rx, next: None, ended: false, children, sink })
|
||||
}
|
||||
|
||||
/// The soundtrack as PCM into rodio (silently skipped when the clip has none).
|
||||
fn spawn_audio(&mut self, start: f64, children: &mut Vec<Child>) -> Option<Sink> {
|
||||
let mut audio = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{start:.3}"), "-i", &self.url, "-vn", "-ac", "2", "-ar", &AUDIO_RATE.to_string(), "-f", "s16le", "-"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().ok()?;
|
||||
let mut stdout = audio.stdout.take()?;
|
||||
children.push(audio);
|
||||
if self.output.is_none() {
|
||||
self.output = OutputStream::try_default().map_err(|e| warn!("No audio output: {e}")).ok();
|
||||
}
|
||||
let (_, handle) = self.output.as_ref()?;
|
||||
let sink = Sink::try_new(handle).ok()?;
|
||||
let (tx, rx) = channel::<Vec<i16>>();
|
||||
std::thread::spawn(move || {
|
||||
let mut buf = vec![0u8; 8192];
|
||||
while let Ok(n) = stdout.read(&mut buf) {
|
||||
if n == 0 || tx.send(buf[..n - n % 2].chunks_exact(2).map(|b| i16::from_le_bytes([b[0], b[1]])).collect()).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
sink.set_volume(if self.muted { 0.0 } else { 1.0 });
|
||||
sink.append(PcmStream { rx, buf: VecDeque::new() });
|
||||
Some(sink)
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads fixed-size RGBA frames from `ffmpeg` and stamps them with their media time.
|
||||
fn read_frames(mut stdout: impl Read, w: u32, h: u32, start: f64, fps: f64, tx: SyncSender<(ColorImage, f64)>) {
|
||||
let mut buf = vec![0u8; (w * h * 4) as usize];
|
||||
let mut n = 0u64;
|
||||
while stdout.read_exact(&mut buf).is_ok() {
|
||||
let image = ColorImage::from_rgba_unmultiplied([w as usize, h as usize], &buf);
|
||||
if tx.send((image, start + n as f64 / fps)).is_err() {
|
||||
break;
|
||||
}
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Live PCM from a channel; silence while it waits, the end when the sender's gone.
|
||||
struct PcmStream {
|
||||
rx: Receiver<Vec<i16>>,
|
||||
buf: VecDeque<i16>,
|
||||
}
|
||||
|
||||
impl Iterator for PcmStream {
|
||||
type Item = i16;
|
||||
|
||||
fn next(&mut self) -> Option<i16> {
|
||||
if self.buf.is_empty() {
|
||||
match self.rx.try_recv() {
|
||||
Ok(chunk) => self.buf.extend(chunk),
|
||||
Err(TryRecvError::Empty) => return Some(0),
|
||||
Err(TryRecvError::Disconnected) => return None,
|
||||
}
|
||||
}
|
||||
self.buf.pop_front().or(Some(0))
|
||||
}
|
||||
}
|
||||
|
||||
impl Source for PcmStream {
|
||||
fn current_frame_len(&self) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
|
||||
fn channels(&self) -> u16 {
|
||||
2
|
||||
}
|
||||
|
||||
fn sample_rate(&self) -> u32 {
|
||||
AUDIO_RATE
|
||||
}
|
||||
|
||||
fn total_duration(&self) -> Option<Duration> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the clip's size, rate and length. A small probe keeps this ~2 s on CDN files instead of
|
||||
/// ~14 s (ffprobe otherwise scans much of the file over HTTP).
|
||||
fn probe(url: &str) -> Result<VideoInfo, String> {
|
||||
#[derive(serde::Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
struct Probe {
|
||||
streams: Vec<ProbeStream>,
|
||||
format: ProbeFormat,
|
||||
}
|
||||
#[derive(serde::Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
struct ProbeStream {
|
||||
width: u32,
|
||||
height: u32,
|
||||
avg_frame_rate: String,
|
||||
}
|
||||
#[derive(serde::Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
struct ProbeFormat {
|
||||
duration: String,
|
||||
}
|
||||
let out = command("ffprobe").args(["-v", "error", "-probesize", "65536", "-analyzeduration", "0", "-select_streams", "v:0", "-show_entries", "stream=width,height,avg_frame_rate:format=duration", "-of", "json", url]).output().map_err(|e| format!("ffprobe: {e}"))?;
|
||||
let probe: Probe = serde_json::from_slice(&out.stdout).map_err(|e| format!("ffprobe: {e}"))?;
|
||||
let stream = probe.streams.first().ok_or("no video stream")?;
|
||||
let fps = match stream.avg_frame_rate.split_once('/') {
|
||||
Some((n, d)) => n.parse::<f64>().unwrap_or(24.0) / d.parse::<f64>().unwrap_or(1.0).max(1.0),
|
||||
None => stream.avg_frame_rate.parse().unwrap_or(24.0),
|
||||
};
|
||||
Ok(VideoInfo { width: stream.width.max(2), height: stream.height.max(2), fps: if fps.is_finite() && fps > 0.0 { fps } else { 24.0 }, duration: probe.format.duration.parse().unwrap_or(0.0) })
|
||||
}
|
||||
|
||||
/// The decoded size: at most `MAX_WIDTH` wide, both sides even (as `scale` requires).
|
||||
fn output_size(info: VideoInfo) -> (u32, u32) {
|
||||
let w = info.width.min(MAX_WIDTH) & !1;
|
||||
let h = ((info.height as f64 * w as f64 / info.width as f64).round() as u32).max(2) & !1;
|
||||
(w, h)
|
||||
}
|
||||
|
||||
/// `m:ss`.
|
||||
fn clock(secs: f64) -> String {
|
||||
let s = secs.max(0.0) as u64;
|
||||
format!("{}:{:02}", s / 60, s % 60)
|
||||
}
|
||||
|
||||
/// A command that doesn't flash a console window on Windows.
|
||||
fn command(program: &str) -> Command {
|
||||
#[allow(unused_mut)]
|
||||
let mut cmd = Command::new(program);
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::process::CommandExt;
|
||||
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||
cmd.creation_flags(CREATE_NO_WINDOW);
|
||||
}
|
||||
cmd
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn output_sizes_are_even_and_capped() {
|
||||
let info = |width, height| VideoInfo { width, height, fps: 24.0, duration: 5.0 };
|
||||
assert_eq!(output_size(info(1920, 1080)), (1280, 720));
|
||||
assert_eq!(output_size(info(721, 1281)), (720, 1278));
|
||||
assert_eq!(clock(65.9), "1:05");
|
||||
}
|
||||
|
||||
/// Needs FFmpeg on the PATH: `cargo test -p artcraft_native -- --ignored`.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn probes_and_decodes_a_generated_clip() {
|
||||
let dir = std::env::temp_dir().join("artcraft-native-video-test");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let clip = dir.join("clip.mp4");
|
||||
let status = command("ffmpeg").args(["-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc=duration=2:size=640x360:rate=24", "-f", "lavfi", "-i", "sine=duration=2", "-shortest", "-pix_fmt", "yuv420p"]).arg(&clip).status().unwrap();
|
||||
assert!(status.success());
|
||||
let url = clip.to_string_lossy().into_owned();
|
||||
let info = probe(&url).unwrap();
|
||||
assert_eq!((info.width, info.height), (640, 360));
|
||||
assert!((info.fps - 24.0).abs() < 0.01 && (info.duration - 2.0).abs() < 0.2);
|
||||
|
||||
let mut player = VideoPlayer { url: url.clone(), probe: None, info: Some(info), stream: None, texture: None, position: 0.0, muted: true, error: None, output: None };
|
||||
let stream = player.spawn(info, 1.0).unwrap();
|
||||
let (frame, pts) = stream.frames.recv_timeout(Duration::from_secs(10)).unwrap();
|
||||
assert_eq!(frame.size, [640, 360]);
|
||||
assert!((pts - 1.0).abs() < 1e-9, "frames are stamped from the start position");
|
||||
let count = 1 + stream.frames.iter().count();
|
||||
assert!((20..=28).contains(&count), "about a second of frames at 24 fps, got {count}");
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,17 @@ impl MediaLinks {
|
||||
}
|
||||
}
|
||||
|
||||
impl MediaLinks {
|
||||
/// A video's animated preview, `width` pixels wide where the CDN can resize.
|
||||
pub fn animated_preview(&self, width: u32) -> Option<String> {
|
||||
let previews = self.maybe_video_previews.as_ref()?;
|
||||
if !previews.animated_thumbnail_template.is_empty() {
|
||||
return Some(previews.animated_thumbnail_template.replace("{WIDTH}", &width.to_string()));
|
||||
}
|
||||
(!previews.animated.is_empty()).then(|| previews.animated.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /v1/media_files/list/user/{username}`.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
@@ -70,10 +81,56 @@ pub struct MediaFile {
|
||||
pub maybe_model_type: Option<String>,
|
||||
pub maybe_origin_model_type: Option<String>,
|
||||
pub maybe_duration_millis: Option<u64>,
|
||||
pub maybe_title: Option<String>,
|
||||
pub media_links: MediaLinks,
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
impl MediaFile {
|
||||
/// The file's still thumbnail; audio files have none (their CDN link is the audio itself).
|
||||
pub fn thumbnail(&self, width: u32) -> Option<String> {
|
||||
if self.media_class.as_deref() == Some("audio") {
|
||||
return None;
|
||||
}
|
||||
self.media_links.thumbnail(width)
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /v1/omni_gen/models/audio`. The client has no binding for it (and the API's type only
|
||||
/// serializes), so it's read here; capability flags are left out when false.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AudioModels {
|
||||
pub success: bool,
|
||||
pub models: Vec<AudioModel>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AudioModel {
|
||||
pub model: String,
|
||||
pub model_creator: Option<String>,
|
||||
pub full_name: Option<String>,
|
||||
pub extra_info_short: Option<String>,
|
||||
pub text_prompt_supported: Option<bool>,
|
||||
pub style_prompt_supported: bool,
|
||||
pub audio_references_supported: bool,
|
||||
pub audio_references_max: Option<u16>,
|
||||
pub image_references_supported: bool,
|
||||
pub image_references_max: Option<u16>,
|
||||
pub keep_lyrics_supported: bool,
|
||||
pub instrumental_toggle_supported: bool,
|
||||
pub loopable_toggle_supported: bool,
|
||||
pub bpm_supported: bool,
|
||||
pub musical_key_supported: bool,
|
||||
pub sample_rate_hz_options: Vec<u32>,
|
||||
pub sample_rate_hz_default: Option<u32>,
|
||||
pub speed_supported: bool,
|
||||
pub volume_supported: bool,
|
||||
pub pitch_supported: bool,
|
||||
pub is_disabled: bool,
|
||||
}
|
||||
|
||||
/// `GET /v1/media_files/batch_gen_redux/{token}`.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
@@ -194,6 +251,36 @@ pub struct ContextImage {
|
||||
pub media_links: MediaLinks,
|
||||
}
|
||||
|
||||
/// A saved character (`GET /v1/characters/session`).
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct Character {
|
||||
pub token: String,
|
||||
pub name: String,
|
||||
pub maybe_description: Option<String>,
|
||||
pub maybe_avatar: Option<MediaLinks>,
|
||||
pub maybe_full_image: Option<MediaLinks>,
|
||||
pub models: Vec<String>,
|
||||
}
|
||||
|
||||
impl Character {
|
||||
pub fn avatar_url(&self) -> Option<String> {
|
||||
self.maybe_avatar.as_ref().or(self.maybe_full_image.as_ref()).and_then(|l| l.thumbnail(256))
|
||||
}
|
||||
|
||||
pub fn full_url(&self) -> Option<String> {
|
||||
self.maybe_full_image.as_ref().or(self.maybe_avatar.as_ref()).map(|l| l.cdn_url.clone()).filter(|u| !u.is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct CharactersPage {
|
||||
pub success: bool,
|
||||
pub characters: Vec<Character>,
|
||||
pub next_cursor: Option<i64>,
|
||||
}
|
||||
|
||||
/// `GET /v1/session` (only the parts the app shows).
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
//! The generation feed as a masonry grid (`GenerationGridView`) or a list (`GenerationListView`):
|
||||
//! running jobs first-class alongside failures and finished media, newest first, loading more as
|
||||
//! the user scrolls.
|
||||
//! the user scrolls. Audio plays in place, with its waveform.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use egui::{Align2, Color32, CornerRadius, FontFamily, FontId, Id, Mesh, Rect, Sense, Stroke, Ui, pos2, vec2};
|
||||
|
||||
use crate::backend::audio::AudioPlayer;
|
||||
use crate::backend::media_cache::{Lookup, MediaCache};
|
||||
use crate::feed::types::{FailedJob, FeedItem, MediaKind, PendingJob, pending_status, time_ago};
|
||||
use crate::theme;
|
||||
use crate::ui::icons::{self, Icon};
|
||||
use crate::ui::{creator_icons, widgets};
|
||||
use crate::ui::{creator_icons, waveform, widgets};
|
||||
|
||||
/// `react-masonry-css` gutter and the cap on tall portraits (≈ 5:7).
|
||||
const GAP: f32 = 8.0;
|
||||
@@ -18,6 +19,8 @@ const MAX_RATIO: f32 = 1.4;
|
||||
/// Start loading the next page this far before the end.
|
||||
const LOAD_MORE_MARGIN: f32 = 400.0;
|
||||
const LIST_THUMB: f32 = 100.0;
|
||||
/// The list's compact audio player is at most `max-w-md` wide.
|
||||
const LIST_PLAYER_MAX: f32 = 448.0;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum ViewMode {
|
||||
@@ -61,6 +64,8 @@ pub struct FeedView<'a> {
|
||||
/// Prompt texts by prompt token (list view and pending cards).
|
||||
pub prompts: &'a HashMap<String, String>,
|
||||
pub make_video: bool,
|
||||
/// Play videos' animated previews instead of their still frames.
|
||||
pub autoplay: bool,
|
||||
pub bottom_padding: f32,
|
||||
}
|
||||
|
||||
@@ -84,7 +89,7 @@ impl Entry<'_> {
|
||||
#[derive(Clone, Default)]
|
||||
pub struct RatioCache(HashMap<String, f32>);
|
||||
|
||||
pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &mut RatioCache, names: &dyn ModelNames) -> Vec<FeedAction> {
|
||||
pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &mut RatioCache, audio: &mut AudioPlayer, names: &dyn ModelNames) -> Vec<FeedAction> {
|
||||
let mut entries: Vec<Entry<'_>> = view.pending.iter().map(Entry::Pending).chain(view.failed.iter().map(Entry::Failed)).chain(view.items.iter().map(Entry::Item)).collect();
|
||||
entries.sort_by_key(|e| std::cmp::Reverse(e.time()));
|
||||
let mut actions = Vec::new();
|
||||
@@ -111,10 +116,10 @@ pub fn show(ui: &mut Ui, view: &FeedView<'_>, cache: &mut MediaCache, ratios: &m
|
||||
match (entry, view.mode) {
|
||||
(Entry::Pending(job), ViewMode::Grid) => pending_card(ui, rect, job, now, names),
|
||||
(Entry::Failed(job), ViewMode::Grid) => failed_card(ui, view.id, rect, job, cache, names, &mut actions),
|
||||
(Entry::Item(item), ViewMode::Grid) => item_card(ui, view, rect, item, cache, ratios, names, &mut actions),
|
||||
(Entry::Item(item), ViewMode::Grid) => item_card(ui, view, rect, item, cache, ratios, audio, names, &mut actions),
|
||||
(Entry::Pending(job), ViewMode::List) => pending_row(ui, rect, job, now, names),
|
||||
(Entry::Failed(job), ViewMode::List) => failed_row(ui, view.id, rect, job, names, &mut actions),
|
||||
(Entry::Item(item), ViewMode::List) => item_row(ui, view, rect, item, cache, names, now, &mut actions),
|
||||
(Entry::Item(item), ViewMode::List) => item_row(ui, view, rect, item, cache, audio, names, now, &mut actions),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,25 +231,15 @@ fn failed_card(ui: &mut Ui, id: Id, rect: Rect, job: &FailedJob, cache: &mut Med
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache, names: &dyn ModelNames, actions: &mut Vec<FeedAction>) {
|
||||
fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache, audio: &mut AudioPlayer, names: &dyn ModelNames, actions: &mut Vec<FeedAction>) {
|
||||
let resp = ui.interact(rect, view.id.with(("item", &item.token)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
let hovered = resp.hovered();
|
||||
ui.painter().rect_filled(rect, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4));
|
||||
match item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
|
||||
Some(Lookup::Ready(t)) => {
|
||||
let size = t.size_vec2();
|
||||
if size.x > 0.0 && !ratios.0.contains_key(&item.token) {
|
||||
ratios.0.insert(item.token.clone(), size.y / size.x);
|
||||
}
|
||||
egui::Image::new(&t).uv(cover_uv_for(size, rect.size())).corner_radius(theme::RADIUS).paint_at(ui, rect);
|
||||
},
|
||||
Some(Lookup::Loading) => shimmer(ui, rect),
|
||||
_ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), if item.kind == MediaKind::Video { Icon::Video } else { Icon::Image }, theme::FAINT),
|
||||
}
|
||||
if item.kind == MediaKind::Video {
|
||||
let badge = Rect::from_center_size(rect.center(), vec2(40.0, 40.0));
|
||||
ui.painter().circle_filled(badge.center(), 20.0, Color32::from_black_alpha(110));
|
||||
icons::paint(ui.painter(), badge.shrink(12.0).translate(vec2(1.5, 0.0)), Icon::Play, theme::fade(Color32::WHITE, 0.9));
|
||||
// The pointer may be on an action button (or the audio player) rather than the card itself.
|
||||
let hovered = resp.hovered() || ui.rect_contains_pointer(rect);
|
||||
let is_audio = item.kind == MediaKind::Audio;
|
||||
if is_audio {
|
||||
audio_face(ui, view, rect, item, audio);
|
||||
} else {
|
||||
visual_face(ui, view, rect, item, cache, ratios);
|
||||
}
|
||||
let selected = view.selected.contains(&item.token);
|
||||
let ring = if selected {
|
||||
@@ -276,25 +271,17 @@ fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cach
|
||||
p.galley(pos2(badge.left() + 20.0, badge.center().y - galley.size().y / 2.0), galley, Color32::WHITE);
|
||||
}
|
||||
let mut action_clicked = false;
|
||||
// The pointer may be on an action button rather than the card itself.
|
||||
if (hovered || ui.rect_contains_pointer(rect)) && !view.selecting {
|
||||
let footer = bottom_gradient(ui, rect, 56.0, 178);
|
||||
let chip_y = footer.bottom() - 18.0;
|
||||
let mut x = footer.left() + 8.0;
|
||||
x = chip(ui, pos2(x, chip_y), Some(if item.kind == MediaKind::Video { Icon::Video } else { Icon::Image }), None, item.kind.label());
|
||||
if hovered && !view.selecting {
|
||||
// Audio keeps its player at the bottom, so its chips and actions go on top.
|
||||
let band = edge_fade(ui, rect, 56.0, 178, is_audio);
|
||||
let chip_y = if is_audio { band.top() + 18.0 } else { band.bottom() - 18.0 };
|
||||
let mut x = band.left() + 8.0;
|
||||
x = chip(ui, pos2(x, chip_y), Some(kind_icon(item.kind)), None, item.kind.label());
|
||||
if let Some(model) = item.model_id.as_deref() {
|
||||
chip(ui, pos2(x + 6.0, chip_y), None, Some(&names.creator(model)), &truncate(&names.display_name(model), 16));
|
||||
}
|
||||
let mut buttons: Vec<(Icon, &str, FeedAction)> = Vec::new();
|
||||
if item.prompt_token.is_some() {
|
||||
buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone())));
|
||||
}
|
||||
if view.make_video && item.kind == MediaKind::Image {
|
||||
buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone())));
|
||||
}
|
||||
buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone())));
|
||||
buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone())));
|
||||
let pill = Rect::from_min_size(pos2(footer.right() - 8.0 - 8.0 - 28.0 * buttons.len() as f32, chip_y - 18.0), vec2(8.0 + 28.0 * buttons.len() as f32, 36.0));
|
||||
let buttons = item_actions(view, item, None);
|
||||
let pill = Rect::from_min_size(pos2(band.right() - 8.0 - 8.0 - 28.0 * buttons.len() as f32, chip_y - 18.0), vec2(8.0 + 28.0 * buttons.len() as f32, 36.0));
|
||||
ui.painter().rect_filled(pill, 0.0, Color32::from_black_alpha(153));
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(pill.shrink(4.0)).layout(egui::Layout::left_to_right(egui::Align::Center)));
|
||||
child.spacing_mut().item_spacing.x = 0.0;
|
||||
@@ -310,6 +297,87 @@ fn item_card(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cach
|
||||
}
|
||||
}
|
||||
|
||||
/// An image or video card's face: the thumbnail (or animated preview) and a video's play badge.
|
||||
fn visual_face(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, ratios: &mut RatioCache) {
|
||||
ui.painter().rect_filled(rect, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4));
|
||||
// Videos play their animated preview when previews are on (the still shows while it loads).
|
||||
let animated = item.animated.as_deref().filter(|_| view.autoplay && item.kind == MediaKind::Video).map(|u| cache.get_animated(ui.ctx(), u));
|
||||
let lookup = match animated {
|
||||
Some(Lookup::Ready(t)) => Some(Lookup::Ready(t)),
|
||||
_ => item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)),
|
||||
};
|
||||
match lookup {
|
||||
Some(Lookup::Ready(t)) => {
|
||||
let size = t.size_vec2();
|
||||
if size.x > 0.0 && !ratios.0.contains_key(&item.token) {
|
||||
ratios.0.insert(item.token.clone(), size.y / size.x);
|
||||
}
|
||||
egui::Image::new(&t).uv(cover_uv_for(size, rect.size())).corner_radius(theme::RADIUS).paint_at(ui, rect);
|
||||
},
|
||||
Some(Lookup::Loading) => shimmer(ui, rect),
|
||||
_ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), kind_icon(item.kind), theme::FAINT),
|
||||
}
|
||||
if item.kind == MediaKind::Video && !(view.autoplay && item.animated.is_some()) {
|
||||
let badge = Rect::from_center_size(rect.center(), vec2(40.0, 40.0));
|
||||
ui.painter().circle_filled(badge.center(), 20.0, Color32::from_black_alpha(110));
|
||||
icons::paint(ui.painter(), badge.shrink(12.0).translate(vec2(1.5, 0.0)), Icon::Play, theme::fade(Color32::WHITE, 0.9));
|
||||
}
|
||||
}
|
||||
|
||||
/// An audio card's face (`GalleryCard`'s audio branch, a square tile): the title, a music note
|
||||
/// and the waveform player along the bottom.
|
||||
fn audio_face(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, audio: &mut AudioPlayer) {
|
||||
let mut mesh = Mesh::default();
|
||||
let (top, bottom) = (theme::fade(theme::ACCENT, 0.14), theme::fade(theme::CONTROLS, 0.6));
|
||||
mesh.colored_vertex(rect.left_top(), top);
|
||||
mesh.colored_vertex(rect.right_top(), top);
|
||||
mesh.colored_vertex(rect.right_bottom(), bottom);
|
||||
mesh.colored_vertex(rect.left_bottom(), bottom);
|
||||
mesh.add_triangle(0, 1, 2);
|
||||
mesh.add_triangle(0, 2, 3);
|
||||
ui.painter().add(mesh);
|
||||
// Clear of the selection checkbox when selecting.
|
||||
let inset = if view.selecting { 36.0 } else { 12.0 };
|
||||
let title = item.prompt_token.as_deref().and_then(|t| view.prompts.get(t)).cloned().unwrap_or_else(|| fallback_title(item));
|
||||
let galley = clamp_lines(ui, &title, 13.0, theme::fade(Color32::WHITE, 0.85), rect.width() - inset - 12.0, 2);
|
||||
ui.painter().galley(pos2(rect.left() + inset, rect.top() + 12.0), galley, Color32::WHITE);
|
||||
waveform::music_tile(ui, Rect::from_center_size(rect.center() - vec2(0.0, 12.0), vec2(48.0, 48.0)), true);
|
||||
let bar = Rect::from_min_max(pos2(rect.left() + 12.0, rect.bottom() - 56.0), pos2(rect.right() - 12.0, rect.bottom() - 12.0));
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar));
|
||||
waveform::player(&mut child, view.id.with(("player", &item.token)), audio, &item.full_url, item.duration_secs, false);
|
||||
}
|
||||
|
||||
/// The hover actions for an item: copy prompt (list rows, when the text is known), recreate,
|
||||
/// make video, share and download. Audio only shares and downloads (`Audio v1`).
|
||||
fn item_actions<'a>(view: &FeedView<'_>, item: &FeedItem, prompt: Option<&String>) -> Vec<(Icon, &'a str, FeedAction)> {
|
||||
let mut buttons = Vec::new();
|
||||
if let Some(text) = prompt {
|
||||
buttons.push((Icon::Copy, "Copy prompt", FeedAction::CopyPrompt(text.clone())));
|
||||
}
|
||||
if item.prompt_token.is_some() && item.kind != MediaKind::Audio {
|
||||
buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone())));
|
||||
}
|
||||
if view.make_video && item.kind == MediaKind::Image {
|
||||
buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone())));
|
||||
}
|
||||
buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone())));
|
||||
buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone())));
|
||||
buttons
|
||||
}
|
||||
|
||||
/// What an item is called without its prompt: its title, else "Audio Generation" and the like.
|
||||
fn fallback_title(item: &FeedItem) -> String {
|
||||
item.title.clone().unwrap_or_else(|| format!("{} Generation", item.kind.label()))
|
||||
}
|
||||
|
||||
fn kind_icon(kind: MediaKind) -> Icon {
|
||||
match kind {
|
||||
MediaKind::Image => Icon::Image,
|
||||
MediaKind::Video => Icon::Video,
|
||||
MediaKind::Audio => Icon::Music,
|
||||
}
|
||||
}
|
||||
|
||||
fn pending_row(ui: &mut Ui, rect: Rect, job: &PendingJob, now: i64, names: &dyn ModelNames) {
|
||||
let thumb = Rect::from_min_size(rect.min + vec2(0.0, 12.0), vec2(LIST_THUMB, LIST_THUMB));
|
||||
ui.painter().rect_filled(thumb, theme::RADIUS, Color32::from_white_alpha(8));
|
||||
@@ -359,7 +427,7 @@ fn failed_row(ui: &mut Ui, id: Id, rect: Rect, job: &FailedJob, names: &dyn Mode
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, names: &dyn ModelNames, now: i64, actions: &mut Vec<FeedAction>) {
|
||||
fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache: &mut MediaCache, audio: &mut AudioPlayer, names: &dyn ModelNames, now: i64, actions: &mut Vec<FeedAction>) {
|
||||
let resp = ui.interact(rect, view.id.with(("row", &item.token)), Sense::click());
|
||||
let hovered = resp.hovered();
|
||||
if hovered {
|
||||
@@ -367,7 +435,10 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
|
||||
}
|
||||
let thumb = Rect::from_min_size(rect.min + vec2(0.0, 12.0), vec2(LIST_THUMB, LIST_THUMB));
|
||||
ui.painter().rect_filled(thumb, theme::RADIUS, theme::fade(theme::CONTROLS, 0.4));
|
||||
if let Some(Lookup::Ready(t)) = item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
|
||||
let is_audio = item.kind == MediaKind::Audio;
|
||||
if is_audio {
|
||||
waveform::music_tile(ui, thumb, false);
|
||||
} else if let Some(Lookup::Ready(t)) = item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
|
||||
egui::Image::new(&t).uv(crate::prompt_box::deck::cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, thumb);
|
||||
}
|
||||
if view.selected.contains(&item.token) {
|
||||
@@ -378,7 +449,8 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
|
||||
let prompt = item.prompt_token.as_deref().and_then(|t| view.prompts.get(t));
|
||||
match prompt {
|
||||
Some(text) => {
|
||||
let g = clamp_lines(ui, text, 14.0, theme::INK, w, 3);
|
||||
// Audio keeps room for its player under the text.
|
||||
let g = clamp_lines(ui, text, 14.0, theme::INK, w, if is_audio { 2 } else { 3 });
|
||||
ui.painter().galley(pos2(x, thumb.top()), g, theme::INK);
|
||||
},
|
||||
None if item.prompt_token.is_some() => {
|
||||
@@ -387,9 +459,13 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
|
||||
}
|
||||
},
|
||||
None => {
|
||||
ui.painter().text(pos2(x, thumb.top() + 8.0), Align2::LEFT_CENTER, format!("{} Generation", item.kind.label()), FontId::new(14.0, FontFamily::Proportional), theme::MUTED);
|
||||
ui.painter().text(pos2(x, thumb.top() + 8.0), Align2::LEFT_CENTER, fallback_title(item), FontId::new(14.0, FontFamily::Proportional), theme::MUTED);
|
||||
},
|
||||
}
|
||||
if is_audio {
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(Rect::from_min_size(pos2(x, thumb.top() + 44.0), vec2(w.min(LIST_PLAYER_MAX), 32.0))));
|
||||
waveform::player(&mut child, view.id.with(("row-player", &item.token)), audio, &item.full_url, item.duration_secs, true);
|
||||
}
|
||||
let meta_y = thumb.bottom() - 10.0;
|
||||
let mut mx = x;
|
||||
if let Some(model) = item.model_id.as_deref() {
|
||||
@@ -400,18 +476,7 @@ fn item_row(ui: &mut Ui, view: &FeedView<'_>, rect: Rect, item: &FeedItem, cache
|
||||
ui.painter().text(pos2(rect.right(), thumb.top() + 8.0), Align2::RIGHT_CENTER, time_ago(item.created_at, now), FontId::new(11.0, FontFamily::Proportional), theme::FAINT);
|
||||
let mut action_clicked = false;
|
||||
if hovered && !view.selecting {
|
||||
let mut buttons: Vec<(Icon, &str, FeedAction)> = Vec::new();
|
||||
if let Some(text) = prompt {
|
||||
buttons.push((Icon::Copy, "Copy prompt", FeedAction::CopyPrompt(text.clone())));
|
||||
}
|
||||
if item.prompt_token.is_some() {
|
||||
buttons.push((Icon::RotateCw, "Recreate", FeedAction::Recreate(item.token.clone())));
|
||||
}
|
||||
if view.make_video && item.kind == MediaKind::Image {
|
||||
buttons.push((Icon::Video, "Make Video", FeedAction::MakeVideo(item.token.clone())));
|
||||
}
|
||||
buttons.push((Icon::Link, "Share", FeedAction::Share(item.token.clone())));
|
||||
buttons.push((Icon::Download, "Download", FeedAction::Download(item.token.clone())));
|
||||
let buttons = item_actions(view, item, prompt);
|
||||
let bar = Rect::from_min_size(pos2(rect.right() - 28.0 * buttons.len() as f32, thumb.bottom() - 28.0), vec2(28.0 * buttons.len() as f32, 28.0));
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar).layout(egui::Layout::left_to_right(egui::Align::Center)));
|
||||
child.spacing_mut().item_spacing.x = 0.0;
|
||||
@@ -464,9 +529,15 @@ fn model_chip(ui: &Ui, left_center: egui::Pos2, model_id: &str, names: &dyn Mode
|
||||
|
||||
/// Darkens the bottom of `rect` (`bg-gradient-to-t from-black/70`), returning the faded band.
|
||||
fn bottom_gradient(ui: &Ui, rect: Rect, height: f32, alpha: u8) -> Rect {
|
||||
let band = Rect::from_min_max(pos2(rect.left(), (rect.bottom() - height).max(rect.top())), rect.max);
|
||||
edge_fade(ui, rect, height, alpha, false)
|
||||
}
|
||||
|
||||
/// Darkens `rect` toward its top or bottom edge over `height`, returning the faded band.
|
||||
fn edge_fade(ui: &Ui, rect: Rect, height: f32, alpha: u8, top_edge: bool) -> Rect {
|
||||
let band = if top_edge { Rect::from_min_max(rect.min, pos2(rect.right(), (rect.top() + height).min(rect.bottom()))) } else { Rect::from_min_max(pos2(rect.left(), (rect.bottom() - height).max(rect.top())), rect.max) };
|
||||
let mut mesh = Mesh::default();
|
||||
let (top, bottom) = (Color32::TRANSPARENT, Color32::from_black_alpha(alpha));
|
||||
let (clear, dark) = (Color32::TRANSPARENT, Color32::from_black_alpha(alpha));
|
||||
let (top, bottom) = if top_edge { (dark, clear) } else { (clear, dark) };
|
||||
mesh.colored_vertex(band.left_top(), top);
|
||||
mesh.colored_vertex(band.right_top(), top);
|
||||
mesh.colored_vertex(band.right_bottom(), bottom);
|
||||
|
||||
@@ -6,12 +6,14 @@ use std::collections::{HashMap, HashSet};
|
||||
use crate::backend::wire::{Job, JobPhase, MediaFile, unix_secs};
|
||||
use crate::backend::EnqueueMeta;
|
||||
use crate::feed::types::{FailedJob, FeedItem, MediaKind, PendingJob};
|
||||
use crate::models::Catalog;
|
||||
use crate::models::{self, Catalog};
|
||||
|
||||
/// Failures older than this are history, not news.
|
||||
const FAILURE_WINDOW_SECS: i64 = 24 * 60 * 60;
|
||||
/// Thumbnails are requested at this width (`getMediaThumbnail(@512)`).
|
||||
pub const THUMBNAIL_WIDTH: u32 = 512;
|
||||
/// Animated previews are requested smaller: every frame becomes a texture.
|
||||
const ANIMATED_WIDTH: u32 = 320;
|
||||
|
||||
/// Something the store wants the app to do or say after an update.
|
||||
#[derive(Debug, PartialEq)]
|
||||
@@ -54,7 +56,7 @@ impl FeedStore {
|
||||
/// Shows the jobs a generate call just started, before the next poll catches up.
|
||||
pub fn add_enqueued(&mut self, kind: MediaKind, job_tokens: &[String], meta: &EnqueueMeta, catalog: &Catalog) {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let expected = catalog.find(&meta.model_id).map_or(30.0, |m| m.expected_secs);
|
||||
let expected = catalog.find(&meta.model_id).map_or_else(|| models::expected_secs(&meta.model_id, kind), |m| m.expected_secs);
|
||||
for token in job_tokens {
|
||||
self.meta.insert(token.clone(), meta.clone());
|
||||
self.seen_running.insert(token.clone());
|
||||
@@ -79,7 +81,7 @@ impl FeedStore {
|
||||
match job.status.phase() {
|
||||
JobPhase::Running => {
|
||||
self.seen_running.insert(job.job_token.clone());
|
||||
let expected = model_id.as_deref().and_then(|id| catalog.find(id)).map_or(if kind == MediaKind::Video { 900.0 } else { 30.0 }, |m| m.expected_secs);
|
||||
let expected = model_id.as_deref().and_then(|id| catalog.find(id)).map_or_else(|| models::expected_secs(model_id.as_deref().unwrap_or_default(), kind), |m| m.expected_secs);
|
||||
pending.push(PendingJob { job_token: job.job_token.clone(), kind, prompt, model_id, created_at: unix_secs(&job.created_at), server_progress: Some(job.status.progress_percentage), expected_secs: expected, batch_count: meta.map_or(1, |m| m.batch_count) });
|
||||
},
|
||||
JobPhase::Failed => {
|
||||
@@ -103,7 +105,8 @@ impl FeedStore {
|
||||
};
|
||||
// The finished card takes the pending card's place: date it by the job.
|
||||
let created_at = unix_secs(&job.created_at);
|
||||
self.push_item(FeedItem { token: result.entity_token.clone(), kind, thumbnail: result.media_links.thumbnail(THUMBNAIL_WIDTH), full_url: result.media_links.cdn_url.clone(), created_at, model_id: model_id.clone(), prompt_token: job.request.maybe_prompt_token.clone(), batch_token: result.maybe_batch_token.clone(), duration_secs: None });
|
||||
let thumbnail = result.media_links.thumbnail(THUMBNAIL_WIDTH).filter(|_| kind != MediaKind::Audio);
|
||||
self.push_item(FeedItem { token: result.entity_token.clone(), kind, title: None, thumbnail, full_url: result.media_links.cdn_url.clone(), created_at, model_id: model_id.clone(), prompt_token: job.request.maybe_prompt_token.clone(), batch_token: result.maybe_batch_token.clone(), duration_secs: None, animated: result.media_links.animated_preview(ANIMATED_WIDTH) });
|
||||
if let Some(batch) = &result.maybe_batch_token {
|
||||
notices.push(FeedNotice::LoadBatch { job_token: job.job_token.clone(), batch_token: batch.clone() });
|
||||
}
|
||||
@@ -207,19 +210,23 @@ pub fn feed_item(kind: MediaKind, file: &MediaFile) -> FeedItem {
|
||||
kind: match file.media_class.as_deref() {
|
||||
Some("video") => MediaKind::Video,
|
||||
Some("image") => MediaKind::Image,
|
||||
Some("audio") => MediaKind::Audio,
|
||||
_ => kind,
|
||||
},
|
||||
thumbnail: file.media_links.thumbnail(THUMBNAIL_WIDTH),
|
||||
title: file.maybe_title.clone().filter(|t| !t.is_empty()),
|
||||
thumbnail: file.thumbnail(THUMBNAIL_WIDTH),
|
||||
full_url: file.media_links.cdn_url.clone(),
|
||||
created_at: unix_secs(&file.created_at),
|
||||
model_id: file.maybe_model_type.clone().or_else(|| file.maybe_origin_model_type.clone()),
|
||||
prompt_token: file.maybe_prompt_token.clone(),
|
||||
batch_token: file.maybe_batch_token.clone(),
|
||||
duration_secs: file.maybe_duration_millis.map(|ms| ms as f32 / 1000.0),
|
||||
animated: file.media_links.animated_preview(ANIMATED_WIDTH),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a job made an image or a video: by its model, else by its category.
|
||||
/// Whether a job made an image, a video or audio: by its model, else by its category
|
||||
/// (`useGenerationJobs` checks video before image before audio).
|
||||
fn job_kind(job: &Job, catalog: &Catalog) -> Option<MediaKind> {
|
||||
if let Some(model) = job.request.maybe_model_type.as_deref() {
|
||||
if catalog.image.iter().any(|m| m.id == model) {
|
||||
@@ -228,12 +235,17 @@ fn job_kind(job: &Job, catalog: &Catalog) -> Option<MediaKind> {
|
||||
if catalog.video.iter().any(|m| m.id == model) {
|
||||
return Some(MediaKind::Video);
|
||||
}
|
||||
if catalog.audio.iter().any(|m| m.id == model) {
|
||||
return Some(MediaKind::Audio);
|
||||
}
|
||||
}
|
||||
let category = job.request.inference_category.as_str();
|
||||
if category.contains("video") {
|
||||
Some(MediaKind::Video)
|
||||
} else if category.contains("image") {
|
||||
Some(MediaKind::Image)
|
||||
} else if category.contains("audio") {
|
||||
Some(MediaKind::Audio)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
pub enum MediaKind {
|
||||
Image,
|
||||
Video,
|
||||
Audio,
|
||||
}
|
||||
|
||||
impl MediaKind {
|
||||
@@ -11,6 +12,7 @@ impl MediaKind {
|
||||
match self {
|
||||
MediaKind::Image => "Image",
|
||||
MediaKind::Video => "Video",
|
||||
MediaKind::Audio => "Audio",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,17 +20,20 @@ impl MediaKind {
|
||||
match self {
|
||||
MediaKind::Image => "images",
|
||||
MediaKind::Video => "videos",
|
||||
MediaKind::Audio => "audio clips",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A finished image or video in the user's library.
|
||||
/// A finished image, video or audio clip in the user's library.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FeedItem {
|
||||
/// The media file token.
|
||||
pub token: String,
|
||||
pub kind: MediaKind,
|
||||
/// A thumbnail URL (512 px wide where the CDN can resize).
|
||||
/// The file's title (`maybe_title`), when it has one.
|
||||
pub title: Option<String>,
|
||||
/// A thumbnail URL (512 px wide where the CDN can resize); audio has none.
|
||||
pub thumbnail: Option<String>,
|
||||
/// The original file.
|
||||
pub full_url: String,
|
||||
@@ -39,6 +44,8 @@ pub struct FeedItem {
|
||||
pub prompt_token: Option<String>,
|
||||
pub batch_token: Option<String>,
|
||||
pub duration_secs: Option<f32>,
|
||||
/// A video's animated preview (shown when previews play).
|
||||
pub animated: Option<String>,
|
||||
}
|
||||
|
||||
/// A generation the server is still working on.
|
||||
|
||||
@@ -8,10 +8,15 @@ use crate::feed::types::MediaKind;
|
||||
/// Default models (`defaultModelForPage.ts`).
|
||||
pub const DEFAULT_IMAGE_MODEL: &str = "nano_banana_pro";
|
||||
pub const DEFAULT_VIDEO_MODEL: &str = "seedance_2p0";
|
||||
pub const DEFAULT_AUDIO_MODEL: &str = "suno_music";
|
||||
/// The fallback prompt limit (`maxPromptLength ?? 1000`).
|
||||
pub const DEFAULT_PROMPT_MAX: usize = 1000;
|
||||
/// The model pickers group by family once they list this many models.
|
||||
const GROUP_THRESHOLD: usize = 8;
|
||||
/// Audio models that work from exactly one audio track (`AUDIO_MODELS_REQUIRING_AUDIO_REF`).
|
||||
const AUDIO_MODELS_REQUIRING_AUDIO_REF: [&str; 2] = ["suno_remix", "suno_sample"];
|
||||
/// The musical keys (`SoundsSettingsPopover`; there are no E keys): value, label, short label.
|
||||
pub const MUSICAL_KEYS: [(&str, &str, &str); 13] = [("auto", "Auto", "Auto"), ("c_major", "C Major", "C"), ("c_minor", "C Minor", "Cm"), ("d_major", "D Major", "D"), ("d_minor", "D Minor", "Dm"), ("f_major", "F Major", "F"), ("f_minor", "F Minor", "Fm"), ("g_major", "G Major", "G"), ("g_minor", "G Minor", "Gm"), ("a_major", "A Major", "A"), ("a_minor", "A Minor", "Am"), ("b_major", "B Major", "B"), ("b_minor", "B Minor", "Bm")];
|
||||
|
||||
/// One generation model and what it accepts.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
@@ -45,6 +50,8 @@ pub struct ModelInfo {
|
||||
pub video_refs_max_secs: Option<u16>,
|
||||
pub audio_refs_max: usize,
|
||||
pub audio_refs_max_secs: Option<u16>,
|
||||
/// `@Character` mentions (0 = not supported).
|
||||
pub character_refs_max: usize,
|
||||
pub duration_options: Vec<u16>,
|
||||
pub duration_min: Option<u16>,
|
||||
pub duration_max: Option<u16>,
|
||||
@@ -54,10 +61,42 @@ pub struct ModelInfo {
|
||||
pub bitrate_default: Option<String>,
|
||||
pub output_formats: Vec<String>,
|
||||
pub output_format_default: Option<String>,
|
||||
/// Audio only.
|
||||
pub audio: AudioCaps,
|
||||
/// How long a generation usually takes (drives the estimated progress bar).
|
||||
pub expected_secs: f32,
|
||||
}
|
||||
|
||||
/// What an audio model takes besides the prompt and references.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct AudioCaps {
|
||||
pub style_prompt: bool,
|
||||
pub instrumental: bool,
|
||||
pub keep_lyrics: bool,
|
||||
pub loopable: bool,
|
||||
pub bpm: bool,
|
||||
pub musical_key: bool,
|
||||
pub sample_rates: Vec<u32>,
|
||||
pub sample_rate_default: Option<u32>,
|
||||
pub speed: bool,
|
||||
pub volume: bool,
|
||||
pub pitch: bool,
|
||||
/// Remixes and samples work from exactly one audio track.
|
||||
pub requires_audio_ref: bool,
|
||||
}
|
||||
|
||||
impl AudioCaps {
|
||||
/// Whether the Tuning popover shows.
|
||||
pub fn has_tuning(&self) -> bool {
|
||||
!self.sample_rates.is_empty() || self.speed || self.volume || self.pitch
|
||||
}
|
||||
|
||||
/// `picked` if the model offers it, else its default, else its first rate.
|
||||
pub fn sample_rate(&self, picked: Option<u32>) -> Option<u32> {
|
||||
picked.filter(|hz| self.sample_rates.contains(hz)).or(self.sample_rate_default).or_else(|| self.sample_rates.first().copied())
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelInfo {
|
||||
/// Batch sizes the count picker offers (`predefined` or `1..=max`).
|
||||
pub fn batch_options(&self) -> Vec<u16> {
|
||||
@@ -162,6 +201,7 @@ pub fn video_model(d: &artcraft_client::endpoints::omni_gen::models::video::omni
|
||||
m.video_refs_max_secs = d.video_references_max_total_duration_seconds;
|
||||
m.audio_refs_max = max(d.audio_references_supported, d.audio_references_max);
|
||||
m.audio_refs_max_secs = d.audio_references_max_total_duration_seconds;
|
||||
m.character_refs_max = max(d.character_references_supported, d.character_references_max);
|
||||
m.aspect_ratios = strings(&d.aspect_ratio_options);
|
||||
m.aspect_default = d.aspect_ratio_default.clone().map(String::from);
|
||||
m.resolutions = strings(&d.resolution_options);
|
||||
@@ -182,6 +222,22 @@ pub fn video_model(d: &artcraft_client::endpoints::omni_gen::models::video::omni
|
||||
m
|
||||
}
|
||||
|
||||
/// Builds a model from the server's audio listing.
|
||||
pub fn audio_model(d: &crate::backend::wire::AudioModel) -> ModelInfo {
|
||||
let mut m = base(&d.model, MediaKind::Audio, d.full_name.as_deref(), d.model_creator.clone());
|
||||
if let Some(short) = d.extra_info_short.as_deref().filter(|s| !s.is_empty()) {
|
||||
m.description = short.to_owned();
|
||||
}
|
||||
m.disabled = d.is_disabled;
|
||||
m.prompt_supported = d.text_prompt_supported.unwrap_or(true);
|
||||
m.prompt_max = None;
|
||||
let max = |supported: bool, max: Option<u16>| if supported { max.map_or(1, usize::from) } else { 0 };
|
||||
m.audio_refs_max = max(d.audio_references_supported, d.audio_references_max);
|
||||
m.image_refs_max = max(d.image_references_supported, d.image_references_max);
|
||||
m.audio = AudioCaps { style_prompt: d.style_prompt_supported, instrumental: d.instrumental_toggle_supported, keep_lyrics: d.keep_lyrics_supported, loopable: d.loopable_toggle_supported, bpm: d.bpm_supported, musical_key: d.musical_key_supported, sample_rates: d.sample_rate_hz_options.clone(), sample_rate_default: d.sample_rate_hz_default, speed: d.speed_supported, volume: d.volume_supported, pitch: d.pitch_supported, requires_audio_ref: AUDIO_MODELS_REQUIRING_AUDIO_REF.contains(&d.model.as_str()) };
|
||||
m
|
||||
}
|
||||
|
||||
fn strings<T: Clone + Into<String>>(options: &Option<Vec<T>>) -> Vec<String> {
|
||||
options.as_deref().unwrap_or_default().iter().cloned().map(Into::into).collect()
|
||||
}
|
||||
@@ -209,7 +265,7 @@ fn base(id: &str, kind: MediaKind, full_name: Option<&str>, creator: Option<Stri
|
||||
|
||||
/// Families whose picker names the app curates (the static model lists' `selectorName`s).
|
||||
fn has_curated_name(id: &str) -> bool {
|
||||
const PREFIXES: [&str; 11] = ["flux_", "nano_banana", "gpt_image", "seedream", "seedance", "kling", "veo_", "sora", "happy_horse", "grok_imagine", "qwen"];
|
||||
const PREFIXES: [&str; 13] = ["flux_", "nano_banana", "gpt_image", "seedream", "seedance", "kling", "veo_", "sora", "happy_horse", "grok_imagine", "qwen", "suno", "seed_audio"];
|
||||
PREFIXES.iter().any(|p| id.starts_with(p))
|
||||
}
|
||||
|
||||
@@ -289,12 +345,17 @@ fn description(id: &str) -> &'static str {
|
||||
"grok_video" | "grok_imagine_video" => "Fastest video model",
|
||||
"sora_2" => "Smart video model",
|
||||
"veo_3" => "Slow, high-quality model",
|
||||
"suno_music" => "Full songs from a text prompt",
|
||||
"suno_remix" => "Remix an existing track",
|
||||
"suno_sounds" => "Sound effects with beat control",
|
||||
"suno_sample" => "Build a song from a sample",
|
||||
"seed_audio_1p0" => "Sound generation with fine tuning",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// Typical generation time (the static list's `progressBarTime`).
|
||||
fn expected_secs(id: &str, kind: MediaKind) -> f32 {
|
||||
/// Typical generation time (the static list's `progressBarTime`, else the feed's per-kind default).
|
||||
pub fn expected_secs(id: &str, kind: MediaKind) -> f32 {
|
||||
match id {
|
||||
"grok_image" | "grok_imagine_image" | "flux_pro_1p1" | "flux_1_dev" | "flux_1_schnell" => 10.0,
|
||||
"grok_video" | "grok_imagine_video" => 50.0,
|
||||
@@ -304,19 +365,20 @@ fn expected_secs(id: &str, kind: MediaKind) -> f32 {
|
||||
"gpt_image_1" | "gpt_image_1p5" | "gpt_image_2p5_flare" | "seedream_4" | "seedream_4p5" | "seedream_5_lite" => 60.0,
|
||||
"gpt_image_2" | "gpt_image_2p5_sunburst" => 120.0,
|
||||
_ if kind == MediaKind::Video => 900.0,
|
||||
_ if kind == MediaKind::Audio => 120.0,
|
||||
_ => 30.0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The maker, by id prefix, when the server doesn't say.
|
||||
pub fn creator_for(id: &str) -> &'static str {
|
||||
const PREFIXES: [(&str, &str); 16] = [("flux", "black_forest_labs"), ("nano_banana", "google"), ("gemini", "google"), ("veo", "google"), ("imagen", "google"), ("gpt_image", "openai"), ("sora", "openai"), ("seedream", "bytedance"), ("seedance", "bytedance"), ("seededit", "bytedance"), ("kling", "kling"), ("grok", "grok"), ("midjourney", "midjourney"), ("qwen", "alibaba"), ("happy_horse", "alibaba"), ("minimax", "minimax")];
|
||||
const PREFIXES: [(&str, &str); 18] = [("suno", "suno"), ("seed_audio", "bytedance"), ("flux", "black_forest_labs"), ("nano_banana", "google"), ("gemini", "google"), ("veo", "google"), ("imagen", "google"), ("gpt_image", "openai"), ("sora", "openai"), ("seedream", "bytedance"), ("seedance", "bytedance"), ("seededit", "bytedance"), ("kling", "kling"), ("grok", "grok"), ("midjourney", "midjourney"), ("qwen", "alibaba"), ("happy_horse", "alibaba"), ("minimax", "minimax")];
|
||||
PREFIXES.iter().find(|(p, _)| id.starts_with(p)).map_or("artcraft", |(_, c)| c)
|
||||
}
|
||||
|
||||
/// The picker family (`getModelFamilyName`).
|
||||
fn family(id: &str) -> &'static str {
|
||||
const FAMILIES: [(&str, &str); 22] = [("seedance", "Seedance"), ("kling", "Kling"), ("veo", "Veo"), ("sora", "Sora"), ("happy_horse", "Happy Horse"), ("wan", "Wan"), ("vidu", "Vidu"), ("preview", "Preview"), ("nano_banana", "Nano Banana"), ("gemini_25_flash", "Nano Banana"), ("seedream", "Seedream"), ("seededit", "SeedEdit"), ("gpt_image", "GPT Image"), ("flux", "Flux"), ("midjourney", "Midjourney"), ("grok", "Grok"), ("qwen", "Qwen"), ("recraft", "Recraft"), ("imagen", "Imagen"), ("minimax", "MiniMax"), ("marble", "World Labs"), ("suno", "Suno")];
|
||||
const FAMILIES: [(&str, &str); 23] = [("seed_audio", "Seed Audio"), ("seedance", "Seedance"), ("kling", "Kling"), ("veo", "Veo"), ("sora", "Sora"), ("happy_horse", "Happy Horse"), ("wan", "Wan"), ("vidu", "Vidu"), ("preview", "Preview"), ("nano_banana", "Nano Banana"), ("gemini_25_flash", "Nano Banana"), ("seedream", "Seedream"), ("seededit", "SeedEdit"), ("gpt_image", "GPT Image"), ("flux", "Flux"), ("midjourney", "Midjourney"), ("grok", "Grok"), ("qwen", "Qwen"), ("recraft", "Recraft"), ("imagen", "Imagen"), ("minimax", "MiniMax"), ("marble", "World Labs"), ("suno", "Suno")];
|
||||
FAMILIES.iter().find(|(p, _)| id.starts_with(p)).map_or("Other", |(_, f)| f)
|
||||
}
|
||||
|
||||
@@ -386,13 +448,16 @@ fn video_page_models(models: &[ModelInfo]) -> Vec<ModelInfo> {
|
||||
list
|
||||
}
|
||||
|
||||
/// Both catalogs, for looking up names in the feed, and each create page's picker list.
|
||||
/// Every catalog, for looking up names in the feed, and each create page's picker list.
|
||||
#[derive(Default)]
|
||||
pub struct Catalog {
|
||||
pub image: Vec<ModelInfo>,
|
||||
pub video: Vec<ModelInfo>,
|
||||
pub audio: Vec<ModelInfo>,
|
||||
pub image_page: Vec<ModelInfo>,
|
||||
pub video_page: Vec<ModelInfo>,
|
||||
/// The audio page's list: enabled, in the server's order.
|
||||
pub audio_page: Vec<ModelInfo>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
@@ -406,8 +471,13 @@ impl Catalog {
|
||||
self.video = models;
|
||||
}
|
||||
|
||||
pub fn set_audio(&mut self, models: Vec<ModelInfo>) {
|
||||
self.audio_page = models.iter().filter(|m| !m.disabled).cloned().collect();
|
||||
self.audio = models;
|
||||
}
|
||||
|
||||
pub fn find(&self, id: &str) -> Option<&ModelInfo> {
|
||||
self.image.iter().chain(&self.video).find(|m| m.id == id)
|
||||
self.image.iter().chain(&self.video).chain(&self.audio).find(|m| m.id == id)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -502,6 +572,15 @@ pub fn quality_label(value: &str) -> String {
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
/// Sample rate labels (`formatSampleRateHz`): 44100 → "44.1 kHz", 24000 → "24 kHz".
|
||||
pub fn sample_rate_label(hz: u32) -> String {
|
||||
if hz.is_multiple_of(1000) {
|
||||
format!("{} kHz", hz / 1000)
|
||||
} else {
|
||||
format!("{:.1} kHz", hz as f32 / 1000.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Bitrate labels.
|
||||
pub fn bitrate_label(value: &str) -> String {
|
||||
match value {
|
||||
@@ -530,6 +609,8 @@ mod tests {
|
||||
assert_eq!(display_name("kling_3p0_pro"), "Kling 3.0 Pro");
|
||||
assert_eq!(display_name("nano_banana"), "Nano Banana");
|
||||
assert_eq!(display_name("seedance_2p0_bpu"), "Seedance 2.0 Plus Ultra");
|
||||
assert_eq!(display_name("seed_audio_1p0"), "Seed Audio 1.0");
|
||||
assert_eq!(display_name("suno_music"), "Suno Music");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -544,6 +625,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_creator_by_prefix() {
|
||||
assert_eq!(creator_for("seed_audio_1p0"), "bytedance");
|
||||
assert_eq!(creator_for("suno_remix"), "suno");
|
||||
assert_eq!(creator_for("seedream_4p5"), "bytedance");
|
||||
assert_eq!(creator_for("veo_3"), "google");
|
||||
assert_eq!(creator_for("something_new"), "artcraft");
|
||||
@@ -573,6 +656,21 @@ mod tests {
|
||||
assert_eq!(ModelInfo::resolve(None, &[], None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_listing_maps_capabilities() {
|
||||
let d: crate::backend::wire::AudioModel = serde_json::from_value(serde_json::json!({ "model": "seed_audio_1p0", "model_creator": "artcraft", "audio_references_supported": true, "audio_references_max": 3, "image_references_supported": true, "sample_rate_hz_options": [8000, 24000, 44100], "sample_rate_hz_default": 24000, "speed_supported": true })).unwrap();
|
||||
let m = audio_model(&d);
|
||||
assert_eq!((m.name.as_str(), m.creator.as_str(), m.description.as_str()), ("Seed Audio 1.0", "bytedance", "Sound generation with fine tuning"));
|
||||
assert_eq!((m.audio_refs_max, m.image_refs_max, m.prompt_max), (3, 1, None));
|
||||
assert!(m.audio.has_tuning() && !m.audio.requires_audio_ref && !m.audio.style_prompt);
|
||||
assert_eq!(m.audio.sample_rate(Some(44100)), Some(44100));
|
||||
assert_eq!(m.audio.sample_rate(Some(11025)), Some(24000), "unsupported picks fall back to the default");
|
||||
let remix = audio_model(&serde_json::from_value(serde_json::json!({ "model": "suno_remix", "audio_references_supported": true })).unwrap());
|
||||
assert!(remix.audio.requires_audio_ref && !remix.audio.has_tuning());
|
||||
assert_eq!((remix.audio_refs_max, remix.audio.sample_rate(None)), (1, None));
|
||||
assert_eq!((sample_rate_label(44100).as_str(), sample_rate_label(24000).as_str()), ("44.1 kHz", "24 kHz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_counts_snap_to_valid_values() {
|
||||
let mut m = model("grok_imagine_image");
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
//! The characters dialog (`CharactersModal`): pick a saved character to mention it in the prompt,
|
||||
//! make a new one from a reference image, rename or re-describe one, or delete it.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use egui::{Align2, Color32, FontId, Id, Rect, Sense, Stroke, Ui, pos2, vec2};
|
||||
|
||||
use crate::backend::UploadedMedia;
|
||||
use crate::backend::media_cache::{Lookup, MediaCache};
|
||||
use crate::backend::wire::Character;
|
||||
use crate::prompt_box::deck::cover_uv;
|
||||
use crate::prompt_box::refs::{RefKind, RefMedia, RefStatus};
|
||||
use crate::theme;
|
||||
use crate::ui::icons::{self, Icon};
|
||||
use crate::ui::widgets;
|
||||
|
||||
const COLUMNS: usize = 4;
|
||||
const GAP: f32 = 12.0;
|
||||
|
||||
#[derive(Clone, PartialEq)]
|
||||
enum View {
|
||||
List,
|
||||
Edit(String),
|
||||
Create,
|
||||
}
|
||||
|
||||
pub struct CharactersModal {
|
||||
view: View,
|
||||
confirm_delete: Option<(String, String)>,
|
||||
name: String,
|
||||
description: String,
|
||||
image: Option<RefMedia>,
|
||||
}
|
||||
|
||||
/// Something only the app can do for the dialog.
|
||||
#[derive(Debug)]
|
||||
pub enum CharactersAction {
|
||||
Close,
|
||||
/// Mention this character in the prompt.
|
||||
Select {
|
||||
token: String,
|
||||
name: String,
|
||||
},
|
||||
Upload {
|
||||
ref_id: u64,
|
||||
path: PathBuf,
|
||||
},
|
||||
Create {
|
||||
image_token: String,
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
},
|
||||
Edit {
|
||||
token: String,
|
||||
name: String,
|
||||
description: String,
|
||||
},
|
||||
Delete {
|
||||
token: String,
|
||||
name: String,
|
||||
},
|
||||
Preview(String),
|
||||
Toast(String),
|
||||
}
|
||||
|
||||
impl CharactersModal {
|
||||
pub fn new() -> Self {
|
||||
Self { view: View::List, confirm_delete: None, name: String::new(), description: String::new(), image: None }
|
||||
}
|
||||
|
||||
/// The reference image being uploaded for a new character, if `ref_id` is it.
|
||||
pub fn apply_upload(&mut self, ref_id: u64, result: Result<UploadedMedia, String>) -> Option<Result<(), String>> {
|
||||
let image = self.image.as_mut().filter(|i| i.id == ref_id)?;
|
||||
Some(match result {
|
||||
Ok(media) => {
|
||||
image.status = RefStatus::Ready;
|
||||
image.token = Some(media.token);
|
||||
image.full_url = media.full_url;
|
||||
Ok(())
|
||||
},
|
||||
Err(err) => {
|
||||
self.image = None;
|
||||
Err(err)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn show(&mut self, ctx: &egui::Context, characters: &[Character], pending: &[String], cache: &mut MediaCache) -> Vec<CharactersAction> {
|
||||
let mut actions = Vec::new();
|
||||
let width = (ctx.content_rect().width() - 80.0).min(760.0);
|
||||
let close = widgets::modal(ctx, Id::new("characters"), width, |ui| match self.view.clone() {
|
||||
View::List => self.list(ui, characters, pending, cache, &mut actions),
|
||||
View::Edit(token) => match characters.iter().find(|c| c.token == token) {
|
||||
Some(c) => self.edit(ui, c, cache, &mut actions),
|
||||
None => self.view = View::List,
|
||||
},
|
||||
View::Create => self.create(ui, cache, &mut actions),
|
||||
});
|
||||
if close && self.confirm_delete.is_none() {
|
||||
actions.push(CharactersAction::Close);
|
||||
}
|
||||
if let Some((token, name)) = self.confirm_delete.clone() {
|
||||
let mut choice = None;
|
||||
let dismissed = widgets::modal(ctx, Id::new("character-delete"), 420.0, |ui| {
|
||||
ui.label(widgets::heading("Delete character?", 20.0));
|
||||
ui.add_space(8.0);
|
||||
ui.label(egui::RichText::new(format!("\u{201c}{name}\u{201d} will be removed from your characters.")).color(theme::MUTED));
|
||||
ui.add_space(16.0);
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
if widgets::button(ui, None, "Delete", widgets::Kind::Danger, 0.0, theme::CONTROL_H).clicked() {
|
||||
choice = Some(true);
|
||||
}
|
||||
if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() {
|
||||
choice = Some(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
if dismissed || choice.is_some() {
|
||||
self.confirm_delete = None;
|
||||
}
|
||||
if choice == Some(true) {
|
||||
actions.push(CharactersAction::Delete { token, name });
|
||||
}
|
||||
}
|
||||
actions
|
||||
}
|
||||
|
||||
fn header(&mut self, ui: &mut Ui, title: &str, back: bool, actions: &mut Vec<CharactersAction>) {
|
||||
ui.horizontal(|ui| {
|
||||
if back && widgets::ghost(ui, Icon::ChevronLeft, None, "Back", false).clicked() {
|
||||
self.view = View::List;
|
||||
}
|
||||
ui.label(widgets::heading(title, 22.0));
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
if widgets::ghost(ui, Icon::X, None, "Close", false).clicked() {
|
||||
actions.push(CharactersAction::Close);
|
||||
}
|
||||
});
|
||||
});
|
||||
ui.add_space(12.0);
|
||||
}
|
||||
|
||||
fn list(&mut self, ui: &mut Ui, characters: &[Character], pending: &[String], cache: &mut MediaCache, actions: &mut Vec<CharactersAction>) {
|
||||
self.header(ui, "Characters", false, actions);
|
||||
let tile = (ui.available_width() - GAP * (COLUMNS - 1) as f32) / COLUMNS as f32;
|
||||
let max_h = ui.ctx().content_rect().height() * 0.6;
|
||||
egui::ScrollArea::vertical().max_height(max_h).auto_shrink([false, true]).show(ui, |ui| {
|
||||
let total = 1 + pending.len() + characters.len();
|
||||
let rows = total.div_ceil(COLUMNS);
|
||||
let origin = ui.cursor().min;
|
||||
ui.allocate_exact_size(vec2(ui.available_width(), rows as f32 * (tile + 30.0 + GAP)), Sense::hover());
|
||||
let cell = |i: usize| Rect::from_min_size(origin + vec2((i % COLUMNS) as f32 * (tile + GAP), (i / COLUMNS) as f32 * (tile + 30.0 + GAP)), vec2(tile, tile));
|
||||
// "Create New" first.
|
||||
let create = cell(0);
|
||||
let resp = ui.interact(create, Id::new("character-create"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, "Create New"));
|
||||
ui.painter().rect_filled(create, theme::RADIUS, theme::fade(theme::CONTROLS, if resp.hovered() { 0.7 } else { 0.4 }));
|
||||
icons::dashed_rect(ui.painter(), create.shrink(1.0), Stroke::new(1.5, theme::LINE_DASHED), 5.0, 4.0);
|
||||
icons::paint(ui.painter(), Rect::from_center_size(create.center() - vec2(0.0, 10.0), vec2(22.0, 22.0)), Icon::Plus, theme::INK);
|
||||
ui.painter().text(create.center() + vec2(0.0, 16.0), Align2::CENTER_CENTER, "Create New", FontId::new(13.0, theme::medium()), theme::INK);
|
||||
if resp.clicked() {
|
||||
self.name.clear();
|
||||
self.description.clear();
|
||||
self.image = None;
|
||||
self.view = View::Create;
|
||||
}
|
||||
for (i, name) in pending.iter().enumerate() {
|
||||
let r = cell(1 + i);
|
||||
ui.painter().rect_filled(r, theme::RADIUS, theme::WASH);
|
||||
icons::paint_spinner(ui, Rect::from_center_size(r.center(), vec2(24.0, 24.0)), theme::MUTED);
|
||||
caption(ui, r, name, "Creating...");
|
||||
}
|
||||
for (i, c) in characters.iter().enumerate() {
|
||||
let r = cell(1 + pending.len() + i);
|
||||
let resp = ui.interact(r, Id::new(("character", &c.token)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, &c.name));
|
||||
thumbnail(ui, r, c.avatar_url().as_deref(), cache);
|
||||
if resp.hovered() {
|
||||
ui.painter().rect_stroke(r, theme::RADIUS, Stroke::new(2.0, theme::fade(theme::ACCENT_400, 0.7)), egui::StrokeKind::Inside);
|
||||
}
|
||||
caption(ui, r, &c.name, "");
|
||||
let mut clicked_tool = false;
|
||||
if ui.rect_contains_pointer(r) {
|
||||
let tools = Rect::from_min_size(pos2(r.right() - 64.0, r.top() + 6.0), vec2(58.0, 28.0));
|
||||
ui.painter().rect_filled(tools, theme::RADIUS, Color32::from_black_alpha(160));
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(tools.shrink(1.0)).layout(egui::Layout::left_to_right(egui::Align::Center)));
|
||||
child.spacing_mut().item_spacing.x = 0.0;
|
||||
if widgets::overlay_icon(&mut child, Icon::Pencil, "Edit character").clicked() {
|
||||
self.name = c.name.clone();
|
||||
self.description = c.maybe_description.clone().unwrap_or_default();
|
||||
self.view = View::Edit(c.token.clone());
|
||||
clicked_tool = true;
|
||||
}
|
||||
if widgets::overlay_icon(&mut child, Icon::Trash, "Delete character").clicked() {
|
||||
self.confirm_delete = Some((c.token.clone(), c.name.clone()));
|
||||
clicked_tool = true;
|
||||
}
|
||||
}
|
||||
if resp.clicked() && !clicked_tool {
|
||||
actions.push(CharactersAction::Select { token: c.token.clone(), name: c.name.clone() });
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn edit(&mut self, ui: &mut Ui, c: &Character, cache: &mut MediaCache, actions: &mut Vec<CharactersAction>) {
|
||||
self.header(ui, "Edit Character", true, actions);
|
||||
ui.horizontal_top(|ui| {
|
||||
ui.vertical(|ui| {
|
||||
let (r, _) = ui.allocate_exact_size(vec2(240.0, 240.0), Sense::hover());
|
||||
thumbnail(ui, r, c.full_url().or_else(|| c.avatar_url()).as_deref(), cache);
|
||||
if let Some(url) = c.full_url() {
|
||||
if ui.add(egui::Button::new(egui::RichText::new("View full size").size(12.0).color(theme::ACCENT_INK)).frame(false)).clicked() {
|
||||
actions.push(CharactersAction::Preview(url));
|
||||
}
|
||||
}
|
||||
});
|
||||
ui.add_space(16.0);
|
||||
ui.vertical(|ui| {
|
||||
fields(ui, &mut self.name, &mut self.description);
|
||||
ui.add_space(16.0);
|
||||
ui.horizontal(|ui| {
|
||||
if widgets::button(ui, None, "Save", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() {
|
||||
if self.name.trim().is_empty() {
|
||||
actions.push(CharactersAction::Toast("Name cannot be empty".to_owned()));
|
||||
} else {
|
||||
actions.push(CharactersAction::Edit { token: c.token.clone(), name: self.name.trim().to_owned(), description: self.description.trim().to_owned() });
|
||||
self.view = View::List;
|
||||
}
|
||||
}
|
||||
if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() {
|
||||
self.view = View::List;
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn create(&mut self, ui: &mut Ui, cache: &mut MediaCache, actions: &mut Vec<CharactersAction>) {
|
||||
self.header(ui, "New Character", true, actions);
|
||||
ui.horizontal_top(|ui| {
|
||||
ui.vertical(|ui| {
|
||||
widgets::hud(ui, "Reference", theme::FAINT);
|
||||
ui.add_space(6.0);
|
||||
let (r, resp) = ui.allocate_exact_size(vec2(240.0, 240.0), Sense::click());
|
||||
match &self.image {
|
||||
Some(img) => {
|
||||
thumbnail(ui, r, img.preview.as_deref(), cache);
|
||||
if img.status == RefStatus::Uploading {
|
||||
ui.painter().rect_filled(r, theme::RADIUS, Color32::from_black_alpha(110));
|
||||
icons::paint_spinner(ui, Rect::from_center_size(r.center(), vec2(28.0, 28.0)), Color32::WHITE);
|
||||
}
|
||||
},
|
||||
None => {
|
||||
ui.painter().rect_filled(r, theme::RADIUS, theme::fade(theme::CONTROLS, if resp.hovered() { 0.7 } else { 0.4 }));
|
||||
icons::dashed_rect(ui.painter(), r.shrink(1.0), Stroke::new(1.5, theme::LINE_DASHED), 5.0, 4.0);
|
||||
icons::paint(ui.painter(), Rect::from_center_size(r.center() - vec2(0.0, 12.0), vec2(24.0, 24.0)), Icon::Upload, theme::INK);
|
||||
ui.painter().text(r.center() + vec2(0.0, 18.0), Align2::CENTER_CENTER, "Upload reference image", FontId::new(13.0, theme::medium()), theme::INK);
|
||||
},
|
||||
}
|
||||
if resp.on_hover_cursor(egui::CursorIcon::PointingHand).clicked() {
|
||||
if let Some(path) = crate::pages::common::pick_files(&[RefKind::Image], false).into_iter().next() {
|
||||
match std::fs::read(&path) {
|
||||
Ok(bytes) => {
|
||||
let mut img = RefMedia::uploading(RefKind::Image, None);
|
||||
let key = format!("local://{}", img.id);
|
||||
cache.insert_bytes(ui.ctx(), &key, Arc::from(bytes));
|
||||
img.preview = Some(key);
|
||||
actions.push(CharactersAction::Upload { ref_id: img.id, path });
|
||||
self.image = Some(img);
|
||||
},
|
||||
Err(err) => actions.push(CharactersAction::Toast(format!("Failed to upload {}: {err}", crate::pages::common::file_name(&path)))),
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
ui.add_space(16.0);
|
||||
ui.vertical(|ui| {
|
||||
fields(ui, &mut self.name, &mut self.description);
|
||||
ui.add_space(16.0);
|
||||
let uploading = self.image.as_ref().is_some_and(|i| i.status == RefStatus::Uploading);
|
||||
ui.horizontal(|ui| {
|
||||
if ui.add_enabled_ui(!uploading, |ui| widgets::button(ui, Some(Icon::Sparkles), "Create", widgets::Kind::Primary, 0.0, theme::CONTROL_H)).inner.clicked() {
|
||||
match (self.name.trim(), self.image.as_ref().and_then(|i| i.token.clone())) {
|
||||
("", _) => actions.push(CharactersAction::Toast("Please enter a character name".to_owned())),
|
||||
(_, None) => actions.push(CharactersAction::Toast("Please upload a reference image".to_owned())),
|
||||
(name, Some(image_token)) => {
|
||||
let description = Some(self.description.trim().to_owned()).filter(|d| !d.is_empty());
|
||||
actions.push(CharactersAction::Create { image_token, name: name.to_owned(), description });
|
||||
self.view = View::List;
|
||||
},
|
||||
}
|
||||
}
|
||||
if widgets::button(ui, None, "Cancel", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() {
|
||||
self.view = View::List;
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// The name and description inputs shared by create and edit.
|
||||
fn fields(ui: &mut Ui, name: &mut String, description: &mut String) {
|
||||
widgets::hud(ui, "Character name", theme::FAINT);
|
||||
ui.add_space(4.0);
|
||||
input_frame(ui, |ui| ui.add(egui::TextEdit::singleline(name).frame(egui::Frame::NONE).desired_width(f32::INFINITY).hint_text(egui::RichText::new("Character name").color(theme::FAINT))));
|
||||
ui.add_space(10.0);
|
||||
widgets::hud(ui, "Description", theme::FAINT);
|
||||
ui.add_space(4.0);
|
||||
input_frame(ui, |ui| ui.add(egui::TextEdit::multiline(description).frame(egui::Frame::NONE).desired_width(f32::INFINITY).desired_rows(4).hint_text(egui::RichText::new("Description...").color(theme::FAINT))));
|
||||
}
|
||||
|
||||
fn input_frame(ui: &mut Ui, add: impl FnOnce(&mut Ui) -> egui::Response) {
|
||||
egui::Frame::new().fill(theme::SUNKEN).stroke(theme::hairline()).corner_radius(theme::RADIUS).inner_margin(egui::Margin::symmetric(10, 8)).show(ui, |ui| add(ui));
|
||||
}
|
||||
|
||||
fn thumbnail(ui: &Ui, rect: Rect, url: Option<&str>, cache: &mut MediaCache) {
|
||||
ui.painter().rect_filled(rect, theme::RADIUS, theme::WASH);
|
||||
match url.map(|u| cache.get(ui.ctx(), u)) {
|
||||
Some(Lookup::Ready(t)) => {
|
||||
egui::Image::new(&t).uv(cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, rect);
|
||||
},
|
||||
Some(Lookup::Loading) => icons::paint_spinner(ui, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), theme::FAINT),
|
||||
_ => icons::paint(ui.painter(), Rect::from_center_size(rect.center(), vec2(28.0, 28.0)), Icon::User, theme::FAINT),
|
||||
}
|
||||
}
|
||||
|
||||
/// The name under a tile (and a muted note after it).
|
||||
fn caption(ui: &Ui, tile: Rect, name: &str, note: &str) {
|
||||
let y = tile.bottom() + 15.0;
|
||||
let galley = ui.painter().layout_no_wrap(name.to_owned(), FontId::new(13.0, theme::medium()), theme::INK);
|
||||
let w = galley.size().x.min(tile.width());
|
||||
ui.painter().with_clip_rect(Rect::from_min_size(pos2(tile.left(), y - 10.0), vec2(tile.width(), 20.0))).galley(pos2(tile.left(), y - galley.size().y / 2.0), galley, theme::INK);
|
||||
if !note.is_empty() {
|
||||
ui.painter().text(pos2(tile.left() + w + 6.0, y), Align2::LEFT_CENTER, note, FontId::new(11.5, egui::FontFamily::Proportional), theme::MUTED);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use egui_kittest::Harness;
|
||||
use egui_kittest::kittest::Queryable;
|
||||
|
||||
struct State {
|
||||
_rt: tokio::runtime::Runtime,
|
||||
cache: MediaCache,
|
||||
modal: CharactersModal,
|
||||
characters: Vec<Character>,
|
||||
actions: Vec<String>,
|
||||
fonts_ready: bool,
|
||||
}
|
||||
|
||||
fn harness() -> Harness<'static, State> {
|
||||
let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().expect("runtime");
|
||||
let cache = MediaCache::new(rt.handle().clone(), reqwest::Client::new());
|
||||
let characters = vec![Character { token: "c1".into(), name: "Mira".into(), ..Default::default() }];
|
||||
let state = State { _rt: rt, cache, modal: CharactersModal::new(), characters, actions: Vec::new(), fonts_ready: false };
|
||||
let mut harness = Harness::builder().with_size(egui::vec2(1000.0, 800.0)).build_ui_state(
|
||||
|ui, s: &mut State| {
|
||||
// The harness draws one frame while building: fonts are installed then, used from the next.
|
||||
if !s.fonts_ready {
|
||||
crate::theme::apply(ui.ctx());
|
||||
s.fonts_ready = true;
|
||||
return;
|
||||
}
|
||||
let actions = s.modal.show(ui.ctx(), &s.characters, &[], &mut s.cache);
|
||||
s.actions.extend(actions.iter().map(|a| format!("{a:?}")));
|
||||
},
|
||||
state,
|
||||
);
|
||||
harness.run();
|
||||
harness
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn picking_a_character_mentions_it() {
|
||||
let mut harness = harness();
|
||||
harness.get_by_label("Mira").click();
|
||||
harness.run();
|
||||
assert!(harness.state().actions.iter().any(|a| a.contains("Select") && a.contains("Mira")), "{:?}", harness.state().actions);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creating_needs_a_name_and_a_reference() {
|
||||
let mut harness = harness();
|
||||
harness.get_by_label("Create New").click();
|
||||
harness.run();
|
||||
harness.get_by_label("Create").click();
|
||||
harness.run();
|
||||
assert_eq!(harness.state().actions, [r#"Toast("Please enter a character name")"#]);
|
||||
harness.get_by_role(egui::accesskit::Role::TextInput).focus();
|
||||
harness.run();
|
||||
harness.get_by_role(egui::accesskit::Role::TextInput).type_text("Mira");
|
||||
harness.run();
|
||||
harness.get_by_label("Create").click();
|
||||
harness.run();
|
||||
assert_eq!(harness.state().actions.last().map(String::as_str), Some(r#"Toast("Please upload a reference image")"#));
|
||||
}
|
||||
}
|
||||
@@ -153,7 +153,7 @@ impl LibraryPicker {
|
||||
}
|
||||
|
||||
fn picks(&self) -> Vec<RefMedia> {
|
||||
self.selected.iter().filter_map(|t| self.items.iter().find(|i| i.token == *t)).map(|f| RefMedia::from_library(self.kind, f.token.clone(), f.media_links.thumbnail(256), Some(f.media_links.cdn_url.clone()), f.maybe_duration_millis.map_or(0.0, |ms| ms as f32 / 1000.0))).collect()
|
||||
self.selected.iter().filter_map(|t| self.items.iter().find(|i| i.token == *t)).map(|f| RefMedia::from_library(self.kind, f.token.clone(), f.thumbnail(256), Some(f.media_links.cdn_url.clone()), f.maybe_duration_millis.map_or(0.0, |ms| ms as f32 / 1000.0))).collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ fn tile_ui(ui: &mut egui::Ui, rect: Rect, item: &MediaFile, kind: RefKind, attac
|
||||
let resp = ui.interact(rect, Id::new(("picker-tile", &item.token)), if attached { Sense::hover() } else { Sense::click() });
|
||||
let p = ui.painter();
|
||||
p.rect_filled(rect, 0.0, theme::fade(theme::CONTROLS, 0.6));
|
||||
match item.media_links.thumbnail(256).map(|u| cache.get(ui.ctx(), &u)) {
|
||||
match item.thumbnail(256).map(|u| cache.get(ui.ctx(), &u)) {
|
||||
Some(Lookup::Ready(t)) => {
|
||||
let image = egui::Image::new(&t).uv(cover_uv(t.size_vec2()));
|
||||
let image = if attached { image.tint(Color32::from_gray(90)) } else { image };
|
||||
|
||||
@@ -1,23 +1,30 @@
|
||||
//! The lightbox (`lightbox-modal`): the media large on black, prev/next through the feed, and a
|
||||
//! side column with the prompt, details and actions.
|
||||
//! The lightbox (`lightbox-modal`): the media large on black (audio as a player panel), prev/next
|
||||
//! through the feed, and a side column with the prompt, details and actions.
|
||||
|
||||
use egui::{Align2, Color32, FontId, Id, Key, Rect, Sense, Ui, pos2, vec2};
|
||||
|
||||
use crate::backend::audio::AudioPlayer;
|
||||
use crate::backend::media_cache::{Lookup, MediaCache};
|
||||
use crate::backend::video::{self, VideoPlayer};
|
||||
use crate::backend::wire::Prompt;
|
||||
use crate::feed::grid::ModelNames;
|
||||
use crate::feed::types::{FeedItem, MediaKind};
|
||||
use crate::theme;
|
||||
use crate::ui::icons::{self, Icon};
|
||||
use crate::ui::{creator_icons, widgets};
|
||||
use crate::ui::{creator_icons, waveform, widgets};
|
||||
|
||||
const SIDE_WIDTH: f32 = 280.0;
|
||||
/// The audio panel's widest (`max-w-xl`) and its padding.
|
||||
const AUDIO_PANEL_MAX: f32 = 576.0;
|
||||
const AUDIO_PANEL_PAD: f32 = 24.0;
|
||||
|
||||
pub struct Lightbox {
|
||||
pub kind: MediaKind,
|
||||
pub token: String,
|
||||
confirm_delete: bool,
|
||||
prompt_expanded: bool,
|
||||
/// The in-app player for videos (when FFmpeg is installed).
|
||||
player: Option<VideoPlayer>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
@@ -35,10 +42,11 @@ pub enum LightboxAction {
|
||||
|
||||
impl Lightbox {
|
||||
pub fn new(kind: MediaKind, token: String) -> Self {
|
||||
Self { kind, token, confirm_delete: false, prompt_expanded: false }
|
||||
Self { kind, token, confirm_delete: false, prompt_expanded: false, player: None }
|
||||
}
|
||||
|
||||
pub fn show(&mut self, ctx: &egui::Context, item: &FeedItem, order: &[String], prompt: Option<&Prompt>, names: &dyn ModelNames, cache: &mut MediaCache) -> Vec<LightboxAction> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn show(&mut self, ctx: &egui::Context, item: &FeedItem, order: &[String], prompt: Option<&Prompt>, names: &dyn ModelNames, cache: &mut MediaCache, audio: &mut AudioPlayer) -> Vec<LightboxAction> {
|
||||
let mut actions = Vec::new();
|
||||
let screen = ctx.content_rect();
|
||||
let size = vec2((screen.width() - 80.0).min(1200.0), (screen.height() - 80.0).min(760.0));
|
||||
@@ -62,7 +70,7 @@ impl Lightbox {
|
||||
ui.horizontal_top(|ui| {
|
||||
let media_rect = Rect::from_min_size(ui.cursor().min, vec2(ui.available_width() - SIDE_WIDTH - 16.0, size.y - 32.0));
|
||||
ui.allocate_rect(media_rect, Sense::hover());
|
||||
self.media(ui, media_rect, item, cache, &prev, &next, &mut actions);
|
||||
self.media(ui, media_rect, item, cache, audio, &prev, &next, &mut actions);
|
||||
ui.add_space(16.0);
|
||||
ui.vertical(|ui| {
|
||||
ui.set_width(SIDE_WIDTH);
|
||||
@@ -80,7 +88,29 @@ impl Lightbox {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn media(&self, ui: &mut Ui, rect: Rect, item: &FeedItem, cache: &mut MediaCache, prev: &Option<String>, next: &Option<String>, actions: &mut Vec<LightboxAction>) {
|
||||
fn media(&mut self, ui: &mut Ui, rect: Rect, item: &FeedItem, cache: &mut MediaCache, audio: &mut AudioPlayer, prev: &Option<String>, next: &Option<String>, actions: &mut Vec<LightboxAction>) {
|
||||
if item.kind == MediaKind::Audio {
|
||||
self.player = None;
|
||||
ui.painter().rect_filled(rect, 0.0, Color32::BLACK);
|
||||
audio_panel(ui, rect, item, audio);
|
||||
self.nav_buttons(ui, rect, prev, next, actions);
|
||||
return;
|
||||
}
|
||||
if item.kind == MediaKind::Video && video::ffmpeg_available() {
|
||||
if self.player.as_ref().is_none_or(|p| p.url() != item.full_url) {
|
||||
self.player = Some(VideoPlayer::new(&item.full_url));
|
||||
}
|
||||
let poster = match item.thumbnail.as_deref().map(|u| cache.get(ui.ctx(), u)) {
|
||||
Some(Lookup::Ready(t)) => Some(t),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(player) = &mut self.player {
|
||||
player.ui(ui, rect, poster.as_ref());
|
||||
}
|
||||
self.nav_buttons(ui, rect, prev, next, actions);
|
||||
return;
|
||||
}
|
||||
self.player = None;
|
||||
ui.painter().rect_filled(rect, 0.0, Color32::BLACK);
|
||||
let url = if item.kind == MediaKind::Image { Some(item.full_url.as_str()) } else { item.thumbnail.as_deref() };
|
||||
match url.map(|u| cache.get(ui.ctx(), u)) {
|
||||
@@ -109,6 +139,11 @@ impl Lightbox {
|
||||
actions.push(LightboxAction::Play(item.full_url.clone()));
|
||||
}
|
||||
}
|
||||
self.nav_buttons(ui, rect, prev, next, actions);
|
||||
}
|
||||
|
||||
/// Previous / next arrows at the media's sides (shown while hovering it).
|
||||
fn nav_buttons(&self, ui: &mut Ui, rect: Rect, prev: &Option<String>, next: &Option<String>, actions: &mut Vec<LightboxAction>) {
|
||||
let hovering = ui.rect_contains_pointer(rect);
|
||||
for (target, left, tip) in [(prev, true, "Previous item"), (next, false, "Next item")] {
|
||||
let Some(token) = target else {
|
||||
@@ -214,7 +249,8 @@ impl Lightbox {
|
||||
|
||||
ui.add_space(16.0);
|
||||
let w = (ui.available_width() - 8.0) / 2.0;
|
||||
if item.prompt_token.is_some() && widgets::button(ui, Some(Icon::RotateCw), "Recreate", widgets::Kind::Primary, ui.available_width(), theme::CONTROL_H).clicked() {
|
||||
// Audio can't be recreated yet (`Audio v1`).
|
||||
if item.prompt_token.is_some() && item.kind != MediaKind::Audio && widgets::button(ui, Some(Icon::RotateCw), "Recreate", widgets::Kind::Primary, ui.available_width(), theme::CONTROL_H).clicked() {
|
||||
actions.push(LightboxAction::Recreate);
|
||||
}
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
@@ -259,6 +295,17 @@ impl Lightbox {
|
||||
}
|
||||
}
|
||||
|
||||
/// Audio in the lightbox: a bordered panel with the music tile and the full-size player.
|
||||
fn audio_panel(ui: &mut Ui, rect: Rect, item: &FeedItem, audio: &mut AudioPlayer) {
|
||||
let width = (rect.width() - 48.0).min(AUDIO_PANEL_MAX);
|
||||
let panel = Rect::from_center_size(rect.center(), vec2(width, AUDIO_PANEL_PAD * 2.0 + 64.0 + 20.0 + 44.0));
|
||||
ui.painter().rect(panel, theme::RADIUS, theme::fade(theme::CONTROLS, 0.6), theme::hairline(), egui::StrokeKind::Inside);
|
||||
waveform::music_tile(ui, Rect::from_center_size(pos2(panel.center().x, panel.top() + AUDIO_PANEL_PAD + 32.0), vec2(64.0, 64.0)), false);
|
||||
let bar = Rect::from_min_max(pos2(panel.left() + AUDIO_PANEL_PAD, panel.bottom() - AUDIO_PANEL_PAD - 44.0), pos2(panel.right() - AUDIO_PANEL_PAD, panel.bottom() - AUDIO_PANEL_PAD));
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(bar));
|
||||
waveform::player(&mut child, Id::new(("lightbox-player", &item.token)), audio, &item.full_url, item.duration_secs, false);
|
||||
}
|
||||
|
||||
fn info_row(ui: &mut Ui, label: &str, value: impl FnOnce(&mut Ui)) {
|
||||
ui.horizontal(|ui| {
|
||||
let (rect, _) = ui.allocate_exact_size(vec2(96.0, 20.0), Sense::hover());
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod account;
|
||||
pub mod characters;
|
||||
pub mod library_picker;
|
||||
pub mod lightbox;
|
||||
|
||||
@@ -7,6 +7,7 @@ use std::sync::Arc;
|
||||
use egui::{Align2, Color32, Id, Mesh, Order, Rect, Ui, pos2, vec2};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::backend::audio::AudioPlayer;
|
||||
use crate::backend::media_cache::MediaCache;
|
||||
use crate::backend::{Backend, Modality, UploadedMedia};
|
||||
use crate::feed::grid::{self, FeedAction, FeedView, RatioCache, ViewMode};
|
||||
@@ -29,28 +30,55 @@ pub struct Env<'a> {
|
||||
pub ctx: egui::Context,
|
||||
pub backend: &'a Backend,
|
||||
pub cache: &'a mut MediaCache,
|
||||
/// Plays reference clips and feed tracks.
|
||||
pub audio: &'a mut AudioPlayer,
|
||||
pub toasts: &'a mut Toasts,
|
||||
pub catalog: &'a Catalog,
|
||||
pub signed_in: bool,
|
||||
pub enter_to_generate: bool,
|
||||
pub view_mode: ViewMode,
|
||||
pub autoplay: bool,
|
||||
pub ratios: &'a mut RatioCache,
|
||||
pub prompts: &'a std::collections::HashMap<String, String>,
|
||||
pub requests: &'a mut Vec<AppRequest>,
|
||||
/// The user's characters, newest first.
|
||||
pub characters: &'a [crate::backend::wire::Character],
|
||||
}
|
||||
|
||||
/// Things only the app can do.
|
||||
#[derive(Debug)]
|
||||
pub enum AppRequest {
|
||||
SignIn,
|
||||
Open { kind: MediaKind, token: String },
|
||||
PickFromLibrary { kind: RefKind, slot: ImageSlot, max: usize, page: MediaKind },
|
||||
MakeVideo { token: String },
|
||||
Recreate { kind: MediaKind, token: String },
|
||||
Open {
|
||||
kind: MediaKind,
|
||||
token: String,
|
||||
},
|
||||
PickFromLibrary {
|
||||
kind: RefKind,
|
||||
slot: ImageSlot,
|
||||
max: usize,
|
||||
page: MediaKind,
|
||||
},
|
||||
MakeVideo {
|
||||
token: String,
|
||||
},
|
||||
Recreate {
|
||||
kind: MediaKind,
|
||||
token: String,
|
||||
},
|
||||
Share(String),
|
||||
Download(Vec<String>),
|
||||
CopyText(String),
|
||||
Preview(String),
|
||||
/// Play or stop an audio reference.
|
||||
ToggleAudio {
|
||||
ref_id: u64,
|
||||
url: String,
|
||||
},
|
||||
/// Open a file in the system's player.
|
||||
PlayExternally(String),
|
||||
/// Open the characters dialog (it mentions the picked one in the video prompt).
|
||||
OpenCharacters,
|
||||
LoadMoreLibrary(FeedKey),
|
||||
}
|
||||
|
||||
@@ -59,6 +87,7 @@ pub enum AppRequest {
|
||||
pub enum FeedKey {
|
||||
Image,
|
||||
Video,
|
||||
Audio,
|
||||
Library,
|
||||
}
|
||||
|
||||
@@ -67,6 +96,7 @@ impl FeedKey {
|
||||
match kind {
|
||||
MediaKind::Image => FeedKey::Image,
|
||||
MediaKind::Video => FeedKey::Video,
|
||||
MediaKind::Audio => FeedKey::Audio,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,9 +105,33 @@ impl FeedKey {
|
||||
match self {
|
||||
FeedKey::Image => "image",
|
||||
FeedKey::Video => "video",
|
||||
FeedKey::Library => "image,video",
|
||||
FeedKey::Audio => "audio",
|
||||
FeedKey::Library => "image,video,audio",
|
||||
}
|
||||
}
|
||||
|
||||
/// What the feed's items default to (the library mixes kinds; each file says its own).
|
||||
pub fn kind(self) -> MediaKind {
|
||||
match self {
|
||||
FeedKey::Image | FeedKey::Library => MediaKind::Image,
|
||||
FeedKey::Video => MediaKind::Video,
|
||||
FeedKey::Audio => MediaKind::Audio,
|
||||
}
|
||||
}
|
||||
|
||||
/// The backend's name for this feed's library requests.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
FeedKey::Image => "image",
|
||||
FeedKey::Video => "video",
|
||||
FeedKey::Audio => "audio",
|
||||
FeedKey::Library => "library",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_name(name: &str) -> Option<Self> {
|
||||
[FeedKey::Image, FeedKey::Video, FeedKey::Audio, FeedKey::Library].into_iter().find(|k| k.name() == name)
|
||||
}
|
||||
}
|
||||
|
||||
/// The page around the feed: the splash empty state (or the feed), a fade behind the prompt box,
|
||||
@@ -87,9 +141,9 @@ pub fn create_shell(ui: &mut Ui, env: &mut Env<'_>, kind: MediaKind, title: &str
|
||||
let area = ui.max_rect();
|
||||
let bottom_offset = *box_height + 40.0;
|
||||
if feed.has_content() {
|
||||
let view = FeedView { id: Id::new(("feed", kind.label())), mode: env.view_mode, pending: &feed.pending, failed: &feed.failed, items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: kind == MediaKind::Image, bottom_padding: bottom_offset + 24.0 };
|
||||
let view = FeedView { id: Id::new(("feed", kind.label())), mode: env.view_mode, pending: &feed.pending, failed: &feed.failed, items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: kind == MediaKind::Image, autoplay: env.autoplay, bottom_padding: bottom_offset + 24.0 };
|
||||
let mut feed_ui = ui.new_child(egui::UiBuilder::new().max_rect(area.shrink2(vec2(12.0, 0.0))));
|
||||
let actions = grid::show(&mut feed_ui, &view, env.cache, env.ratios, env.catalog);
|
||||
let actions = grid::show(&mut feed_ui, &view, env.cache, env.ratios, env.audio, env.catalog);
|
||||
handle_feed_actions(env, kind, feed, actions);
|
||||
bottom_fade(ui, area);
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,882 @@
|
||||
//! Create Audio (`CreateAudio` + `PromptBoxAudio`): a song, sound or sample from a prompt, with
|
||||
//! the model's options (style, instrumental, lyrics, loop, beat & key, tuning), audio or image
|
||||
//! references, the cost, and the audio feed.
|
||||
|
||||
use std::ops::RangeInclusive;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use egui::{Align2, Color32, FontId, Id, Rect, Response, Sense, Stroke, TextureHandle, Ui, pos2, vec2};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::backend::media_cache::Lookup;
|
||||
use crate::backend::{EnqueueMeta, Modality};
|
||||
use crate::feed::store::FeedStore;
|
||||
use crate::feed::types::MediaKind;
|
||||
use crate::models::{self, AudioCaps, Catalog, DEFAULT_AUDIO_MODEL, MUSICAL_KEYS, ModelInfo};
|
||||
use crate::pages::common::{self, AppRequest, CostState, Env};
|
||||
use crate::prompt_box::deck::cover_uv;
|
||||
use crate::prompt_box::model_selector::model_selector;
|
||||
use crate::prompt_box::pickers;
|
||||
use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, RefStatus, References};
|
||||
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot};
|
||||
use crate::theme;
|
||||
use crate::ui::icons::{self, Icon};
|
||||
use crate::ui::toast::Toasts;
|
||||
use crate::ui::widgets::{self, Leading};
|
||||
|
||||
const PLACEHOLDER: &str = "Describe the music or sound you want...";
|
||||
const STYLE_PLACEHOLDER: &str = "Style: e.g. dreamy synth-pop, female vocals";
|
||||
/// The audio references' total length cap (`AUDIO_REF_MAX_DURATION_SECONDS`).
|
||||
const MAX_REF_SECS: f32 = 600.0;
|
||||
/// Beat & Key and Tuning (`SoundsSettingsPopover`, `AudioTuningPopover`).
|
||||
const BPM_RANGE: RangeInclusive<u16> = 40..=240;
|
||||
const DEFAULT_BPM: u16 = 120;
|
||||
const SPEED_RANGE: RangeInclusive<f32> = 0.5..=2.0;
|
||||
const VOLUME_RANGE: RangeInclusive<f32> = 0.5..=2.0;
|
||||
const PITCH_RANGE: RangeInclusive<i8> = -12..=12;
|
||||
const POPOVER_WIDTH: f32 = 240.0;
|
||||
/// Shown when generate fails on the server's side (`OMNI_GENERATE_OUTAGE_MESSAGE`).
|
||||
const OUTAGE_MESSAGE: &str = "Couldn't start generation - the service is temporarily unavailable. Try again shortly.";
|
||||
|
||||
/// What the page remembers between launches.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct AudioSettings {
|
||||
pub prompt: String,
|
||||
pub style_prompt: String,
|
||||
pub model: Option<String>,
|
||||
pub instrumental: bool,
|
||||
pub keep_lyrics: bool,
|
||||
pub loopable: bool,
|
||||
/// `None` = Auto.
|
||||
pub bpm: Option<u16>,
|
||||
pub musical_key: String,
|
||||
/// `None` = the model's default.
|
||||
pub sample_rate_hz: Option<u32>,
|
||||
pub speed: f32,
|
||||
pub volume: f32,
|
||||
/// Semitones.
|
||||
pub pitch: i8,
|
||||
}
|
||||
|
||||
impl Default for AudioSettings {
|
||||
fn default() -> Self {
|
||||
Self { prompt: String::new(), style_prompt: String::new(), model: None, instrumental: false, keep_lyrics: false, loopable: false, bpm: None, musical_key: "auto".to_owned(), sample_rate_hz: None, speed: 1.0, volume: 1.0, pitch: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CreateAudioPage {
|
||||
pub settings: AudioSettings,
|
||||
pub refs: References,
|
||||
pub feed: FeedStore,
|
||||
pub cost: CostState,
|
||||
pub generating: bool,
|
||||
box_state: PromptBoxState,
|
||||
box_height: f32,
|
||||
}
|
||||
|
||||
/// What the reference row asks for.
|
||||
#[derive(Debug, PartialEq)]
|
||||
enum RowAction {
|
||||
AddAudio,
|
||||
AudioFromLibrary,
|
||||
AddImage,
|
||||
Remove(u64),
|
||||
Toggle(u64),
|
||||
}
|
||||
|
||||
impl CreateAudioPage {
|
||||
pub fn new(settings: AudioSettings) -> Self {
|
||||
let mut page = Self { settings, box_height: 138.0, ..Default::default() };
|
||||
page.box_state.editor.focus_requested = true;
|
||||
page
|
||||
}
|
||||
|
||||
pub fn ui(&mut self, ui: &mut Ui, env: &mut Env<'_>) {
|
||||
let model = common::selected_model(&env.catalog.audio_page, self.settings.model.as_deref(), DEFAULT_AUDIO_MODEL).cloned();
|
||||
if let Some(m) = &model {
|
||||
let fields = self.fields(m, true);
|
||||
self.cost.update(ui, env.backend, Modality::Audio, &fields);
|
||||
}
|
||||
let mut feed = std::mem::take(&mut self.feed);
|
||||
let mut box_height = self.box_height;
|
||||
common::create_shell(ui, env, MediaKind::Audio, "Create Audio", "Describe a song, a sound, or a sample", &mut feed, &mut box_height, |ui, env| self.prompt_box(ui, env, model.as_ref()));
|
||||
self.feed = feed;
|
||||
self.box_height = box_height;
|
||||
}
|
||||
|
||||
fn prompt_box(&mut self, ui: &mut Ui, env: &mut Env<'_>, model: Option<&ModelInfo>) {
|
||||
let missing_ref = model.is_some_and(|m| self.missing_required_ref(m));
|
||||
let accepts: Vec<RefKind> = model.map_or_else(Vec::new, |m| [(RefKind::Audio, m.audio_refs_max), (RefKind::Image, m.image_refs_max)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect());
|
||||
let props = PromptBoxProps { id: Id::new("audio-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: None, deck: DeckMode::None, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !missing_ref, generating: self.generating, generate_tooltip: "Generate", warning: missing_ref.then_some("Audio track required"), banner: None, extra_input: !self.settings.style_prompt.is_empty() };
|
||||
|
||||
let playing = self.refs.audios.iter().map(|r| r.id).find(|id| env.audio.is_playing(&ref_key(*id)));
|
||||
let image = self.refs.images.first().and_then(|r| r.preview.as_deref()).and_then(|key| match env.cache.get(&env.ctx, key) {
|
||||
Lookup::Ready(t) => Some(t),
|
||||
_ => None,
|
||||
});
|
||||
let models = &env.catalog.audio_page;
|
||||
let mut prompt = std::mem::take(&mut self.settings.prompt);
|
||||
let settings = &mut self.settings;
|
||||
let refs = &self.refs;
|
||||
let mut picked_model = None;
|
||||
let mut row_actions = Vec::new();
|
||||
let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) {
|
||||
(Slot::ToolbarLeft, _) => {
|
||||
if let Some(id) = model_selector(ui, Id::new("audio-model"), models, model, false) {
|
||||
picked_model = Some(id);
|
||||
}
|
||||
if let Some(m) = model {
|
||||
audio_toolbar(ui, &m.audio, settings);
|
||||
}
|
||||
},
|
||||
(Slot::AbovePrompt, Some(m)) => reference_row(ui, m, refs, playing, image.as_ref(), &mut row_actions),
|
||||
(Slot::BelowPrompt, Some(m)) if m.audio.style_prompt => style_row(ui, &mut settings.style_prompt),
|
||||
_ => {},
|
||||
};
|
||||
let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots);
|
||||
self.settings.prompt = prompt;
|
||||
if let Some(id) = picked_model {
|
||||
self.settings.model = Some(id);
|
||||
}
|
||||
if let Some(m) = model {
|
||||
for action in row_actions {
|
||||
self.row_action(action, env, m);
|
||||
}
|
||||
}
|
||||
for action in actions {
|
||||
self.handle(action, env, model);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(&mut self, action: PromptBoxAction, env: &mut Env<'_>, model: Option<&ModelInfo>) {
|
||||
match action {
|
||||
PromptBoxAction::Generate => self.generate(env, model),
|
||||
// Toggles and tuning stay as they are.
|
||||
PromptBoxAction::ClearAll => {
|
||||
self.settings.prompt.clear();
|
||||
self.settings.style_prompt.clear();
|
||||
self.refs.clear();
|
||||
},
|
||||
PromptBoxAction::DroppedFiles(paths) => {
|
||||
if let Some(m) = model {
|
||||
self.add_dropped(env, m, &paths);
|
||||
}
|
||||
},
|
||||
PromptBoxAction::PastedImage(png) => match model {
|
||||
Some(m) if m.image_refs_max > 0 => {
|
||||
self.make_room_for_image(env);
|
||||
common::upload_png(env, &mut self.refs, png, ImageSlot::Reference);
|
||||
},
|
||||
_ => env.toasts.error("Image references aren't available here"),
|
||||
},
|
||||
PromptBoxAction::Deck(_) | PromptBoxAction::MentionPicked(..) => {},
|
||||
}
|
||||
}
|
||||
|
||||
fn row_action(&mut self, action: RowAction, env: &mut Env<'_>, m: &ModelInfo) {
|
||||
match action {
|
||||
RowAction::AddAudio => {
|
||||
let paths = common::pick_files(&[RefKind::Audio], m.audio_refs_max > 1);
|
||||
self.add_audio(env, m, &paths);
|
||||
},
|
||||
RowAction::AudioFromLibrary => {
|
||||
let max = m.audio_refs_max.saturating_sub(self.refs.audios.len()).max(1);
|
||||
env.requests.push(AppRequest::PickFromLibrary { kind: RefKind::Audio, slot: ImageSlot::Reference, max, page: MediaKind::Audio });
|
||||
},
|
||||
RowAction::AddImage => {
|
||||
if let Some(path) = common::pick_files(&[RefKind::Image], false).first() {
|
||||
self.make_room_for_image(env);
|
||||
common::upload_path(env, &mut self.refs, path, RefKind::Image, ImageSlot::Reference);
|
||||
}
|
||||
},
|
||||
RowAction::Remove(id) => {
|
||||
if env.audio.is_playing(&ref_key(id)) {
|
||||
env.audio.stop();
|
||||
}
|
||||
self.refs.remove(id);
|
||||
},
|
||||
RowAction::Toggle(id) => {
|
||||
if let Some(url) = self.refs.audios.iter().find(|r| r.id == id).and_then(|r| r.full_url.clone()) {
|
||||
env.requests.push(AppRequest::ToggleAudio { ref_id: id, url });
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Dropped files (`usePromptBoxDrop`): audio wins over images; only the first image counts.
|
||||
fn add_dropped(&mut self, env: &mut Env<'_>, m: &ModelInfo, paths: &[PathBuf]) {
|
||||
let audios: Vec<PathBuf> = paths.iter().filter(|p| RefKind::from_path(p) == Some(RefKind::Audio)).cloned().collect();
|
||||
let image = paths.iter().find(|p| RefKind::from_path(p) == Some(RefKind::Image));
|
||||
if !audios.is_empty() {
|
||||
self.add_audio(env, m, &audios);
|
||||
} else if let Some(path) = image {
|
||||
if m.image_refs_max == 0 {
|
||||
env.toasts.error("Image references aren't available here");
|
||||
return;
|
||||
}
|
||||
self.make_room_for_image(env);
|
||||
common::upload_path(env, &mut self.refs, path, RefKind::Image, ImageSlot::Reference);
|
||||
} else if !paths.is_empty() {
|
||||
env.toasts.error(format!("Only {} files can be added here", accepted_kinds(m)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Uploads audio tracks into the free slots (dropping an image reference: they don't mix).
|
||||
fn add_audio(&mut self, env: &mut Env<'_>, m: &ModelInfo, paths: &[PathBuf]) {
|
||||
if paths.is_empty() {
|
||||
return;
|
||||
}
|
||||
if m.audio_refs_max == 0 {
|
||||
env.toasts.error("Audio references aren't available here");
|
||||
return;
|
||||
}
|
||||
let free = m.audio_refs_max.saturating_sub(self.refs.audios.len());
|
||||
if paths.len() > free {
|
||||
env.toasts.error(format!("Max {} audio track{}", m.audio_refs_max, if m.audio_refs_max == 1 { "" } else { "s" }));
|
||||
}
|
||||
if free == 0 {
|
||||
return;
|
||||
}
|
||||
self.make_room_for_audio(env.toasts);
|
||||
for path in paths.iter().take(free) {
|
||||
common::upload_path(env, &mut self.refs, path, RefKind::Audio, ImageSlot::Reference);
|
||||
}
|
||||
}
|
||||
|
||||
/// Library picks, ready to attach: audio within the slots and the total-length cap.
|
||||
pub fn attach_from_library(&mut self, toasts: &mut Toasts, catalog: &Catalog, kind: RefKind, picks: Vec<RefMedia>) {
|
||||
let Some(m) = common::selected_model(&catalog.audio_page, self.settings.model.as_deref(), DEFAULT_AUDIO_MODEL) else {
|
||||
return;
|
||||
};
|
||||
if kind != RefKind::Audio || picks.is_empty() {
|
||||
common::attach_from_library(&mut self.refs, kind, ImageSlot::Reference, picks);
|
||||
return;
|
||||
}
|
||||
self.make_room_for_audio(toasts);
|
||||
let mut total = self.refs.total_secs(RefKind::Audio);
|
||||
let mut accepted = Vec::new();
|
||||
for pick in picks.into_iter().take(m.audio_refs_max.saturating_sub(self.refs.audios.len())) {
|
||||
if total + pick.duration_secs > MAX_REF_SECS {
|
||||
toasts.error(duration_message());
|
||||
break;
|
||||
}
|
||||
total += pick.duration_secs;
|
||||
accepted.push(pick);
|
||||
}
|
||||
common::attach_from_library(&mut self.refs, kind, ImageSlot::Reference, accepted);
|
||||
}
|
||||
|
||||
/// After an upload lands: keep the audio references under the total-length cap.
|
||||
pub fn check_durations(&mut self, toasts: &mut Toasts, ref_id: u64) {
|
||||
if self.refs.audios.iter().any(|r| r.id == ref_id) && self.refs.total_secs(RefKind::Audio) > MAX_REF_SECS + 0.5 {
|
||||
self.refs.remove(ref_id);
|
||||
toasts.error(duration_message());
|
||||
}
|
||||
}
|
||||
|
||||
/// Audio and image references don't mix: adding audio drops the image.
|
||||
fn make_room_for_audio(&mut self, toasts: &mut Toasts) {
|
||||
if !self.refs.images.is_empty() {
|
||||
self.refs.images.clear();
|
||||
toasts.error("Removed image reference \u{2014} it can't be combined with audio");
|
||||
}
|
||||
}
|
||||
|
||||
/// One image at most, and never with audio.
|
||||
fn make_room_for_image(&mut self, env: &mut Env<'_>) {
|
||||
if !self.refs.audios.is_empty() {
|
||||
if self.refs.audios.iter().any(|r| env.audio.is_playing(&ref_key(r.id))) {
|
||||
env.audio.stop();
|
||||
}
|
||||
self.refs.audios.clear();
|
||||
env.toasts.error("Removed audio reference \u{2014} it can't be combined with an image");
|
||||
}
|
||||
self.refs.images.clear();
|
||||
}
|
||||
|
||||
fn missing_required_ref(&self, m: &ModelInfo) -> bool {
|
||||
m.audio.requires_audio_ref && self.refs.audios.len() != 1
|
||||
}
|
||||
|
||||
fn generate(&mut self, env: &mut Env<'_>, model: Option<&ModelInfo>) {
|
||||
let Some(m) = model.filter(|_| !self.generating && !self.settings.prompt.trim().is_empty()) else {
|
||||
return;
|
||||
};
|
||||
if self.missing_required_ref(m) {
|
||||
env.toasts.error(format!("{} needs an audio track to work from \u{2014} add one first", m.name));
|
||||
return;
|
||||
}
|
||||
if !env.signed_in {
|
||||
env.requests.push(AppRequest::SignIn);
|
||||
return;
|
||||
}
|
||||
if self.refs.any_uploading() {
|
||||
env.toasts.error("Please wait for your references to finish uploading");
|
||||
return;
|
||||
}
|
||||
self.generating = true;
|
||||
let meta = EnqueueMeta { prompt: self.settings.prompt.trim().to_owned(), model_id: m.id.clone(), batch_count: 1, ref_image: None };
|
||||
env.backend.generate(Modality::Audio, self.fields(m, false), meta);
|
||||
}
|
||||
|
||||
pub fn on_enqueued(&mut self, env: &mut Env<'_>, job_tokens: &[String], meta: &EnqueueMeta) {
|
||||
self.generating = false;
|
||||
self.feed.add_enqueued(MediaKind::Audio, job_tokens, meta, env.catalog);
|
||||
env.toasts.success("Audio generation enqueued!");
|
||||
}
|
||||
|
||||
pub fn on_enqueue_failed(&mut self, env: &mut Env<'_>, message: &str, status: Option<u16>) {
|
||||
self.generating = false;
|
||||
env.toasts.error(failure_message(message, status));
|
||||
}
|
||||
|
||||
/// The OmniGen request (`buildAudioGenerationBody`): only what the model takes, nothing null.
|
||||
/// The estimate counts references with placeholders and needs no prompt.
|
||||
fn fields(&self, m: &ModelInfo, estimate: bool) -> Map<String, Value> {
|
||||
let (s, caps) = (&self.settings, &m.audio);
|
||||
let mut f = Map::new();
|
||||
f.insert("model".into(), json!(m.id));
|
||||
let tokens = |list: &[RefMedia]| -> Vec<String> {
|
||||
if estimate {
|
||||
vec!["placeholder".to_owned(); list.len()]
|
||||
} else {
|
||||
list.iter().filter_map(|r| r.token.clone()).collect()
|
||||
}
|
||||
};
|
||||
for (key, max, list) in [("audio_media_tokens", m.audio_refs_max, &self.refs.audios), ("image_media_tokens", m.image_refs_max, &self.refs.images)] {
|
||||
let list = tokens(list.as_slice());
|
||||
if max > 0 && !list.is_empty() {
|
||||
f.insert(key.into(), json!(list));
|
||||
}
|
||||
}
|
||||
if let Some(hz) = caps.sample_rate(s.sample_rate_hz).filter(|_| !caps.sample_rates.is_empty()) {
|
||||
f.insert("sample_rate_hz".into(), json!(hz));
|
||||
}
|
||||
if estimate {
|
||||
return f;
|
||||
}
|
||||
let (prompt, style) = (s.prompt.trim(), s.style_prompt.trim());
|
||||
if !prompt.is_empty() {
|
||||
f.insert("prompt".into(), json!(prompt));
|
||||
}
|
||||
if caps.style_prompt && !style.is_empty() {
|
||||
f.insert("style_prompt".into(), json!(style));
|
||||
}
|
||||
for (supported, key, value) in [(caps.keep_lyrics, "keep_lyrics", s.keep_lyrics), (caps.instrumental, "is_instrumental", s.instrumental), (caps.loopable, "is_loopable", s.loopable)] {
|
||||
if supported {
|
||||
f.insert(key.into(), json!(value));
|
||||
}
|
||||
}
|
||||
if let Some(bpm) = s.bpm.filter(|_| caps.bpm) {
|
||||
f.insert("bpm".into(), json!(bpm));
|
||||
}
|
||||
if caps.musical_key {
|
||||
f.insert("musical_key".into(), json!(musical_key(&s.musical_key).0));
|
||||
}
|
||||
for (supported, key, value) in [(caps.speed, "speed", s.speed), (caps.volume, "volume", s.volume)] {
|
||||
if supported {
|
||||
f.insert(key.into(), json!(hundredths(value)));
|
||||
}
|
||||
}
|
||||
if caps.pitch {
|
||||
f.insert("pitch".into(), json!(s.pitch));
|
||||
}
|
||||
f
|
||||
}
|
||||
}
|
||||
|
||||
/// The audio pickers after the model (`PromptBoxAudio`'s toolbar): the toggles the model offers,
|
||||
/// Beat & Key, and Tuning.
|
||||
fn audio_toolbar(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) {
|
||||
let toggles = [(caps.instrumental, &mut s.instrumental, "audio-instrumental", Icon::MicOff, "Instrumental"), (caps.keep_lyrics, &mut s.keep_lyrics, "audio-keep-lyrics", Icon::Mic, "Keep lyrics"), (caps.loopable, &mut s.loopable, "audio-loop", Icon::Repeat, "Loop")];
|
||||
for (supported, value, id, icon, label) in toggles {
|
||||
if supported && pickers::toggle(ui, Id::new(id), icon, label, *value, &format!("{label}: {}", if *value { "ON" } else { "OFF" })) {
|
||||
*value = !*value;
|
||||
}
|
||||
}
|
||||
if caps.bpm || caps.musical_key {
|
||||
beat_and_key(ui, caps, s);
|
||||
}
|
||||
if caps.has_tuning() {
|
||||
tuning(ui, caps, s);
|
||||
}
|
||||
}
|
||||
|
||||
/// The Beat & Key popover: a BPM slider with Auto, and the musical key grid.
|
||||
fn beat_and_key(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) {
|
||||
let mut parts = Vec::new();
|
||||
if caps.bpm {
|
||||
parts.push(s.bpm.map_or_else(|| "Auto BPM".to_owned(), |b| format!("{b} BPM")));
|
||||
}
|
||||
if caps.musical_key {
|
||||
parts.push(musical_key(&s.musical_key).2.to_owned());
|
||||
}
|
||||
let label = if parts.is_empty() { "Beat".to_owned() } else { parts.join(" \u{b7} ") };
|
||||
pickers::popover_pill(ui, Id::new("audio-beat"), Leading::Icon(Icon::Drum), &label, "Beat & Key", |ui| {
|
||||
ui.set_width(POPOVER_WIDTH);
|
||||
widgets::menu_header(ui, "Beat & Key");
|
||||
padded(ui, |ui| {
|
||||
if caps.bpm {
|
||||
ui.horizontal(|ui| {
|
||||
setting_name(ui, "BPM");
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
setting_value(ui, &s.bpm.map_or_else(|| "Auto".to_owned(), |b| b.to_string()));
|
||||
if s.bpm.is_some() && ui.add(egui::Button::new(egui::RichText::new("Auto").size(11.5).color(theme::ACCENT_INK)).frame(false)).on_hover_cursor(egui::CursorIcon::PointingHand).clicked() {
|
||||
s.bpm = None;
|
||||
}
|
||||
});
|
||||
});
|
||||
let mut bpm = s.bpm.unwrap_or(DEFAULT_BPM);
|
||||
if full_slider(ui, &mut bpm, BPM_RANGE, 1.0).changed() {
|
||||
s.bpm = Some(bpm);
|
||||
}
|
||||
}
|
||||
if caps.musical_key {
|
||||
ui.add_space(6.0);
|
||||
setting_name(ui, "Musical key");
|
||||
ui.spacing_mut().item_spacing = vec2(4.0, 4.0);
|
||||
let width = ui.available_width();
|
||||
let (auto, keys) = MUSICAL_KEYS.split_first().expect("keys");
|
||||
if choice_chip(ui, auto.2, s.musical_key == auto.0, width, auto.1).clicked() {
|
||||
s.musical_key = auto.0.to_owned();
|
||||
}
|
||||
let cell = (width - 12.0) / 4.0;
|
||||
for row in keys.chunks(4) {
|
||||
ui.horizontal(|ui| {
|
||||
for (value, label, short) in row {
|
||||
if choice_chip(ui, short, s.musical_key == *value, cell, label).clicked() {
|
||||
s.musical_key = (*value).to_owned();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// The Tuning popover: sample rate chips and the speed, volume and pitch sliders.
|
||||
fn tuning(ui: &mut Ui, caps: &AudioCaps, s: &mut AudioSettings) {
|
||||
pickers::popover_pill(ui, Id::new("audio-tuning"), Leading::Icon(Icon::SlidersHorizontal), "Tuning", "Tuning", |ui| {
|
||||
ui.set_width(POPOVER_WIDTH);
|
||||
widgets::menu_header(ui, "Tuning");
|
||||
padded(ui, |ui| {
|
||||
if !caps.sample_rates.is_empty() {
|
||||
setting_name(ui, "Sample rate");
|
||||
let current = caps.sample_rate(s.sample_rate_hz);
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
ui.spacing_mut().item_spacing = vec2(4.0, 4.0);
|
||||
for hz in &caps.sample_rates {
|
||||
let label = models::sample_rate_label(*hz);
|
||||
if choice_chip(ui, &label, current == Some(*hz), 72.0, &label).clicked() {
|
||||
s.sample_rate_hz = Some(*hz);
|
||||
}
|
||||
}
|
||||
});
|
||||
ui.add_space(6.0);
|
||||
}
|
||||
if caps.speed {
|
||||
labeled_slider(ui, "Speed", &format!("{:.2}\u{d7}", s.speed), &mut s.speed, SPEED_RANGE, 0.05);
|
||||
}
|
||||
if caps.volume {
|
||||
labeled_slider(ui, "Volume", &format!("{:.2}\u{d7}", s.volume), &mut s.volume, VOLUME_RANGE, 0.05);
|
||||
}
|
||||
if caps.pitch {
|
||||
labeled_slider(ui, "Pitch", &format!("{}{} st", if s.pitch > 0 { "+" } else { "" }, s.pitch), &mut s.pitch, PITCH_RANGE, 1.0);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// The references above the prompt (`AudioReferenceRow`): the audio tracks (play, remove), add or
|
||||
/// pick more while there's room, and the image when the model takes one.
|
||||
fn reference_row(ui: &mut Ui, m: &ModelInfo, refs: &References, playing: Option<u64>, image: Option<&TextureHandle>, actions: &mut Vec<RowAction>) {
|
||||
if m.audio_refs_max == 0 && m.image_refs_max == 0 {
|
||||
return;
|
||||
}
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
ui.spacing_mut().item_spacing = vec2(8.0, 6.0);
|
||||
if m.audio_refs_max > 0 {
|
||||
icons::icon(ui, Icon::Music, 14.0, theme::MUTED);
|
||||
ui.label(egui::RichText::new(format!("Audio Track ({}/{})", refs.audios.len(), m.audio_refs_max)).size(12.5).color(theme::MUTED));
|
||||
if m.audio.requires_audio_ref && refs.audios.is_empty() {
|
||||
ui.label(egui::RichText::new("required").size(12.5).color(theme::BAD));
|
||||
}
|
||||
for (i, r) in refs.audios.iter().enumerate() {
|
||||
audio_tile(ui, i, r, playing == Some(r.id), actions);
|
||||
}
|
||||
if refs.audios.len() < m.audio_refs_max {
|
||||
let uploading = refs.audios.iter().any(|r| r.status == RefStatus::Uploading);
|
||||
if ui.add_enabled_ui(!uploading, |ui| widgets::pill(ui, Leading::Icon(Icon::Upload), if uploading { "Uploading\u{2026}" } else { "Add audio" }, false)).inner.clicked() {
|
||||
actions.push(RowAction::AddAudio);
|
||||
}
|
||||
if widgets::pill(ui, Leading::Icon(Icon::Folder), "From library", false).clicked() {
|
||||
actions.push(RowAction::AudioFromLibrary);
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.image_refs_max > 0 {
|
||||
ui.add_space(8.0);
|
||||
ui.label(egui::RichText::new("Image").size(12.5).color(theme::MUTED));
|
||||
match refs.images.first() {
|
||||
Some(r) => image_tile(ui, r, image, actions),
|
||||
None => {
|
||||
if widgets::pill(ui, Leading::Icon(Icon::Image), "Add image", false).clicked() {
|
||||
actions.push(RowAction::AddImage);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
});
|
||||
ui.add_space(8.0);
|
||||
hairline(ui);
|
||||
ui.add_space(8.0);
|
||||
}
|
||||
|
||||
/// One audio reference (`AudioRefTile`): play / stop, "Audio 1 · 12s", remove.
|
||||
fn audio_tile(ui: &mut Ui, index: usize, r: &RefMedia, playing: bool, actions: &mut Vec<RowAction>) {
|
||||
let uploading = r.status == RefStatus::Uploading;
|
||||
let text = if uploading { format!("Audio {}", index + 1) } else { format!("Audio {} \u{b7} {}s", index + 1, r.duration_secs.round() as i64) };
|
||||
let galley = ui.painter().layout_no_wrap(text, FontId::new(12.5, theme::medium()), theme::INK);
|
||||
let (rect, _) = ui.allocate_exact_size(vec2(6.0 + 24.0 + 8.0 + galley.size().x + 6.0 + 20.0 + 6.0, theme::CONTROL_H), Sense::hover());
|
||||
ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, Stroke::new(1.0, theme::LINE), egui::StrokeKind::Inside);
|
||||
let play = Rect::from_center_size(pos2(rect.left() + 18.0, rect.center().y), vec2(24.0, 24.0));
|
||||
if uploading {
|
||||
icons::paint_spinner(ui, play.shrink(4.0), theme::MUTED);
|
||||
} else {
|
||||
let resp = ui.interact(play, Id::new(("audio-ref-play", r.id)), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
let label = if playing { "Stop" } else { "Play" };
|
||||
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
|
||||
let fill = match (playing, resp.hovered()) {
|
||||
(true, _) => theme::ACCENT,
|
||||
(false, true) => theme::WASH_HOVER,
|
||||
(false, false) => theme::WASH,
|
||||
};
|
||||
ui.painter().circle_filled(play.center(), 12.0, fill);
|
||||
if playing {
|
||||
ui.painter().rect_filled(Rect::from_center_size(play.center(), vec2(8.0, 8.0)), 1.0, Color32::WHITE);
|
||||
} else {
|
||||
icons::paint(ui.painter(), play.shrink(7.0).translate(vec2(1.0, 0.0)), Icon::Play, theme::INK);
|
||||
}
|
||||
if resp.clicked() {
|
||||
actions.push(RowAction::Toggle(r.id));
|
||||
}
|
||||
}
|
||||
ui.painter().galley(pos2(play.right() + 8.0, rect.center().y - galley.size().y / 2.0), galley, theme::INK);
|
||||
let remove = Rect::from_center_size(pos2(rect.right() - 16.0, rect.center().y), vec2(20.0, 20.0));
|
||||
if x_button(ui, remove, Id::new(("audio-ref-remove", r.id)), "Remove audio", true) {
|
||||
actions.push(RowAction::Remove(r.id));
|
||||
}
|
||||
}
|
||||
|
||||
/// The image reference: a 36 px thumbnail with a remove × on hover.
|
||||
fn image_tile(ui: &mut Ui, r: &RefMedia, texture: Option<&TextureHandle>, actions: &mut Vec<RowAction>) {
|
||||
let (rect, _) = ui.allocate_exact_size(vec2(36.0, 36.0), Sense::hover());
|
||||
ui.painter().rect(rect, theme::RADIUS, theme::CONTROLS, Stroke::new(1.0, theme::LINE), egui::StrokeKind::Inside);
|
||||
if let Some(t) = texture {
|
||||
egui::Image::new(t).uv(cover_uv(t.size_vec2())).corner_radius(theme::RADIUS).paint_at(ui, rect.shrink(1.0));
|
||||
}
|
||||
if r.status == RefStatus::Uploading {
|
||||
icons::paint_spinner(ui, rect.shrink(9.0), Color32::WHITE);
|
||||
}
|
||||
let remove = Rect::from_min_size(pos2(rect.right() - 14.0, rect.top() - 4.0), vec2(18.0, 18.0));
|
||||
let shown = ui.rect_contains_pointer(rect.union(remove));
|
||||
if x_button(ui, remove, Id::new(("image-ref-remove", r.id)), "Remove image", shown) {
|
||||
actions.push(RowAction::Remove(r.id));
|
||||
}
|
||||
}
|
||||
|
||||
/// The style line under the prompt (`StylePromptRow`).
|
||||
fn style_row(ui: &mut Ui, style: &mut String) {
|
||||
ui.add_space(6.0);
|
||||
hairline(ui);
|
||||
ui.add_space(6.0);
|
||||
ui.horizontal(|ui| {
|
||||
icons::icon(ui, Icon::Tags, 14.0, theme::DIM);
|
||||
let clear_w = if style.is_empty() { 0.0 } else { 28.0 };
|
||||
ui.add(egui::TextEdit::singleline(style).id_salt("audio-style").hint_text(STYLE_PLACEHOLDER).frame(egui::Frame::NONE).font(FontId::new(14.0, egui::FontFamily::Proportional)).desired_width(ui.available_width() - clear_w));
|
||||
if !style.is_empty() {
|
||||
let (rect, _) = ui.allocate_exact_size(vec2(20.0, 20.0), Sense::hover());
|
||||
if x_button(ui, rect, Id::new("audio-style-clear"), "Clear style", true) {
|
||||
style.clear();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// A small round × at `rect`, painted while `shown` (it stays clickable for keyboards and tests).
|
||||
fn x_button(ui: &mut Ui, rect: Rect, id: Id, label: &str, shown: bool) -> bool {
|
||||
let resp = ui.interact(rect, id, Sense::click());
|
||||
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
|
||||
if shown || resp.hovered() {
|
||||
let fill = if resp.hovered() { theme::fade(theme::DANGER, 0.7) } else { Color32::from_black_alpha(128) };
|
||||
ui.painter().circle_filled(rect.center(), rect.width() / 2.0, fill);
|
||||
icons::paint(ui.painter(), rect.shrink(rect.width() * 0.27), Icon::X, Color32::WHITE);
|
||||
}
|
||||
resp.on_hover_text(label).on_hover_cursor(egui::CursorIcon::PointingHand).clicked()
|
||||
}
|
||||
|
||||
/// A selectable chip in the popovers (keys, sample rates): outlined, filled when selected.
|
||||
fn choice_chip(ui: &mut Ui, label: &str, selected: bool, width: f32, tooltip: &str) -> Response {
|
||||
let (rect, resp) = ui.allocate_exact_size(vec2(width, 28.0), Sense::click());
|
||||
let (fill, border) = match (selected, resp.hovered()) {
|
||||
(true, _) => (theme::WASH_HOVER, theme::fade(Color32::WHITE, 0.8)),
|
||||
(false, true) => (theme::WASH, theme::LINE_STRONG),
|
||||
(false, false) => (Color32::TRANSPARENT, theme::LINE),
|
||||
};
|
||||
ui.painter().rect(rect, theme::RADIUS, fill, Stroke::new(1.0, border), egui::StrokeKind::Inside);
|
||||
ui.painter().text(rect.center(), Align2::CENTER_CENTER, label, FontId::new(12.5, theme::medium()), if selected { theme::INK } else { theme::MUTED });
|
||||
resp.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Button, true, selected, label));
|
||||
resp.on_hover_text(tooltip).on_hover_cursor(egui::CursorIcon::PointingHand)
|
||||
}
|
||||
|
||||
/// A setting's name, its value at the right, and a slider across the popover.
|
||||
fn labeled_slider<N: egui::emath::Numeric>(ui: &mut Ui, name: &str, value: &str, number: &mut N, range: RangeInclusive<N>, step: f64) {
|
||||
ui.horizontal(|ui| {
|
||||
setting_name(ui, name);
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| setting_value(ui, value));
|
||||
});
|
||||
full_slider(ui, number, range, step);
|
||||
ui.add_space(4.0);
|
||||
}
|
||||
|
||||
fn full_slider<N: egui::emath::Numeric>(ui: &mut Ui, number: &mut N, range: RangeInclusive<N>, step: f64) -> Response {
|
||||
ui.style_mut().spacing.slider_width = ui.available_width();
|
||||
ui.add(egui::Slider::new(number, range).step_by(step).show_value(false))
|
||||
}
|
||||
|
||||
fn setting_name(ui: &mut Ui, name: &str) {
|
||||
ui.label(egui::RichText::new(name).size(12.0).color(theme::MUTED));
|
||||
}
|
||||
|
||||
fn setting_value(ui: &mut Ui, value: &str) {
|
||||
ui.label(egui::RichText::new(value).size(12.0).family(theme::semibold()).color(theme::INK));
|
||||
}
|
||||
|
||||
fn padded(ui: &mut Ui, content: impl FnOnce(&mut Ui)) {
|
||||
egui::Frame::new().inner_margin(egui::Margin::symmetric(8, 6)).show(ui, content);
|
||||
}
|
||||
|
||||
fn hairline(ui: &mut Ui) {
|
||||
let (line, _) = ui.allocate_exact_size(vec2(ui.available_width(), 1.0), Sense::hover());
|
||||
ui.painter().rect_filled(line, 0.0, theme::fade(Color32::WHITE, 0.1));
|
||||
}
|
||||
|
||||
/// The key a stored value names (Auto for anything unknown): value, label, short label.
|
||||
fn musical_key(value: &str) -> &'static (&'static str, &'static str, &'static str) {
|
||||
MUSICAL_KEYS.iter().find(|k| k.0 == value).unwrap_or(&MUSICAL_KEYS[0])
|
||||
}
|
||||
|
||||
/// The audio player's key for a reference clip.
|
||||
fn ref_key(id: u64) -> String {
|
||||
format!("ref:{id}")
|
||||
}
|
||||
|
||||
/// A slider's value to two decimals (0.05 steps drift in `f32`).
|
||||
fn hundredths(value: f32) -> f64 {
|
||||
(f64::from(value) * 100.0).round() / 100.0
|
||||
}
|
||||
|
||||
/// "audio", "audio or image" (for the drop toasts).
|
||||
fn accepted_kinds(m: &ModelInfo) -> String {
|
||||
let kinds: Vec<&str> = [("audio", m.audio_refs_max), ("image", m.image_refs_max)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect();
|
||||
if kinds.is_empty() {
|
||||
"reference".to_owned()
|
||||
} else {
|
||||
kinds.join(" or ")
|
||||
}
|
||||
}
|
||||
|
||||
fn duration_message() -> String {
|
||||
format!("Total audio duration cannot exceed {}s", MAX_REF_SECS as u32)
|
||||
}
|
||||
|
||||
/// What a failed generate tells the user: out of credits, an outage, or the server's message.
|
||||
fn failure_message(message: &str, status: Option<u16>) -> String {
|
||||
match status {
|
||||
Some(402) => "Not enough credits for this generation".to_owned(),
|
||||
Some(code) if code >= 500 => OUTAGE_MESSAGE.to_owned(),
|
||||
_ if message.is_empty() => "Failed to start audio generation".to_owned(),
|
||||
_ => message.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn model(json: Value) -> ModelInfo {
|
||||
models::audio_model(&serde_json::from_value(json).expect("audio model"))
|
||||
}
|
||||
|
||||
/// The server's audio models (`configs/omni_gen/audio_models.rs`).
|
||||
fn suno_music() -> ModelInfo {
|
||||
model(json!({ "model": "suno_music", "style_prompt_supported": true, "instrumental_toggle_supported": true }))
|
||||
}
|
||||
|
||||
fn suno_sounds() -> ModelInfo {
|
||||
model(json!({ "model": "suno_sounds", "loopable_toggle_supported": true, "bpm_supported": true, "musical_key_supported": true }))
|
||||
}
|
||||
|
||||
fn suno_remix() -> ModelInfo {
|
||||
model(json!({ "model": "suno_remix", "style_prompt_supported": true, "keep_lyrics_supported": true, "audio_references_supported": true, "audio_references_max": 1 }))
|
||||
}
|
||||
|
||||
fn seed_audio() -> ModelInfo {
|
||||
model(json!({ "model": "seed_audio_1p0", "audio_references_supported": true, "audio_references_max": 3, "image_references_supported": true, "image_references_max": 1, "sample_rate_hz_options": [8000, 16000, 24000, 32000, 44100, 48000], "sample_rate_hz_default": 24000, "speed_supported": true, "volume_supported": true, "pitch_supported": true }))
|
||||
}
|
||||
|
||||
fn page(settings: AudioSettings) -> CreateAudioPage {
|
||||
CreateAudioPage { settings, ..Default::default() }
|
||||
}
|
||||
|
||||
fn audio_ref(token: &str, secs: f32) -> RefMedia {
|
||||
RefMedia::from_library(RefKind::Audio, token.into(), None, Some(format!("https://cdn/{token}.mp3")), secs)
|
||||
}
|
||||
|
||||
mod request_bodies {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn suno_music_sends_only_what_it_takes() {
|
||||
let p = page(AudioSettings { prompt: " lofi beat ".into(), style_prompt: " chill ".into(), instrumental: true, keep_lyrics: true, bpm: Some(90), ..Default::default() });
|
||||
assert_eq!(Value::Object(p.fields(&suno_music(), false)), json!({ "model": "suno_music", "prompt": "lofi beat", "style_prompt": "chill", "is_instrumental": true }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suno_sounds_sends_its_toggles_beat_and_key() {
|
||||
let mut p = page(AudioSettings { prompt: "door creak".into(), ..Default::default() });
|
||||
assert_eq!(Value::Object(p.fields(&suno_sounds(), false)), json!({ "model": "suno_sounds", "prompt": "door creak", "is_loopable": false, "musical_key": "auto" }), "Auto BPM is left to the model");
|
||||
p.settings.bpm = Some(128);
|
||||
p.settings.musical_key = "c_minor".into();
|
||||
let f = p.fields(&suno_sounds(), false);
|
||||
assert_eq!((f["bpm"].clone(), f["musical_key"].clone()), (json!(128), json!("c_minor")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_audio_sends_references_and_tuning() {
|
||||
let mut p = page(AudioSettings { prompt: "rain".into(), speed: 1.05, volume: 0.5, pitch: -3, sample_rate_hz: Some(11025), ..Default::default() });
|
||||
p.refs.audios = vec![audio_ref("mf_a", 10.0), audio_ref("mf_b", 20.0)];
|
||||
assert_eq!(Value::Object(p.fields(&seed_audio(), false)), json!({ "model": "seed_audio_1p0", "prompt": "rain", "audio_media_tokens": ["mf_a", "mf_b"], "sample_rate_hz": 24000, "speed": 1.05, "volume": 0.5, "pitch": -3 }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimates_count_references_without_the_prompt() {
|
||||
let mut p = page(AudioSettings { prompt: "rain".into(), sample_rate_hz: Some(44100), ..Default::default() });
|
||||
p.refs.audios = vec![RefMedia::uploading(RefKind::Audio, None), audio_ref("mf_b", 5.0)];
|
||||
assert_eq!(Value::Object(p.fields(&seed_audio(), true)), json!({ "model": "seed_audio_1p0", "audio_media_tokens": ["placeholder", "placeholder"], "sample_rate_hz": 44100 }));
|
||||
assert_eq!(Value::Object(p.fields(&suno_music(), true)), json!({ "model": "suno_music" }), "references the model can't take don't count");
|
||||
}
|
||||
}
|
||||
|
||||
mod rules {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn remixes_need_exactly_one_track() {
|
||||
let mut p = page(AudioSettings::default());
|
||||
assert!(p.missing_required_ref(&suno_remix()));
|
||||
assert!(!p.missing_required_ref(&suno_music()));
|
||||
p.refs.audios.push(audio_ref("mf_a", 30.0));
|
||||
assert!(!p.missing_required_ref(&suno_remix()));
|
||||
p.refs.audios.push(audio_ref("mf_b", 30.0));
|
||||
assert!(p.missing_required_ref(&suno_remix()), "two tracks are too many");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn library_picks_respect_slots_the_cap_and_drop_the_image() {
|
||||
let mut catalog = Catalog::default();
|
||||
catalog.set_audio(vec![seed_audio()]);
|
||||
let mut p = page(AudioSettings { model: Some("seed_audio_1p0".into()), ..Default::default() });
|
||||
p.refs.images.push(RefMedia::from_library(RefKind::Image, "mf_img".into(), None, None, 0.0));
|
||||
let mut toasts = Toasts::default();
|
||||
p.attach_from_library(&mut toasts, &catalog, RefKind::Audio, vec![audio_ref("mf_a", 400.0), audio_ref("mf_b", 300.0), audio_ref("mf_c", 10.0)]);
|
||||
let tokens: Vec<_> = p.refs.audios.iter().filter_map(|r| r.token.as_deref()).collect();
|
||||
assert_eq!(tokens, ["mf_a"], "the second would pass 600s, so picking stops there");
|
||||
assert!(p.refs.images.is_empty(), "audio and an image don't mix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploads_past_the_cap_are_dropped() {
|
||||
let mut p = page(AudioSettings::default());
|
||||
p.refs.audios = vec![audio_ref("mf_a", 590.0), audio_ref("mf_b", 30.0)];
|
||||
let late = p.refs.audios[1].id;
|
||||
p.check_durations(&mut Toasts::default(), late);
|
||||
assert_eq!(p.refs.audios.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failures_explain_credits_and_outages() {
|
||||
assert_eq!(failure_message("Payment required", Some(402)), "Not enough credits for this generation");
|
||||
assert_eq!(failure_message("boom", Some(503)), OUTAGE_MESSAGE);
|
||||
assert_eq!(failure_message("Prompt too long", Some(400)), "Prompt too long");
|
||||
assert_eq!(failure_message("", None), "Failed to start audio generation");
|
||||
}
|
||||
}
|
||||
|
||||
/// The toolbar headlessly (egui_kittest): toggles, Beat & Key and Tuning.
|
||||
mod toolbar {
|
||||
use egui_kittest::Harness;
|
||||
use egui_kittest::kittest::{NodeT, Queryable};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct State {
|
||||
caps: AudioCaps,
|
||||
settings: AudioSettings,
|
||||
fonts_ready: bool,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toggles_show_per_model_and_flip() {
|
||||
let mut harness = build(suno_music().audio);
|
||||
assert!(harness.query_by_label("Loop").is_none(), "Suno Music has no loop toggle");
|
||||
harness.get_by_label("Instrumental").click();
|
||||
harness.run();
|
||||
assert!(harness.state().settings.instrumental);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn beat_and_key_sets_the_key_and_bpm() {
|
||||
let mut harness = build(suno_sounds().audio);
|
||||
harness.get_by_label("Auto BPM \u{b7} Auto").click();
|
||||
harness.run();
|
||||
harness.get_by_label("Cm").click();
|
||||
harness.run();
|
||||
assert_eq!(harness.state().settings.musical_key, "c_minor");
|
||||
assert!(harness.query_by_label("Auto BPM \u{b7} Cm").is_some(), "the pill shows the key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tuning_picks_a_sample_rate() {
|
||||
let mut harness = build(seed_audio().audio);
|
||||
harness.get_by_label("Tuning").click();
|
||||
harness.run();
|
||||
// egui reports a selected button as toggled.
|
||||
assert!(harness.query_by_label("24 kHz").is_some_and(|n| n.accesskit_node().toggled() == Some(egui::accesskit::Toggled::True)), "the default is selected");
|
||||
harness.get_by_label("44.1 kHz").click();
|
||||
harness.run();
|
||||
assert_eq!(harness.state().settings.sample_rate_hz, Some(44100));
|
||||
}
|
||||
|
||||
fn build(caps: AudioCaps) -> Harness<'static, State> {
|
||||
let mut harness = Harness::builder().with_size(egui::vec2(1000.0, 600.0)).build_ui_state(
|
||||
|ui, s: &mut State| {
|
||||
if !s.fonts_ready {
|
||||
crate::theme::apply(ui.ctx());
|
||||
s.fonts_ready = true;
|
||||
return;
|
||||
}
|
||||
ui.add_space(400.0);
|
||||
ui.horizontal(|ui| audio_toolbar(ui, &s.caps, &mut s.settings));
|
||||
},
|
||||
State { caps, settings: AudioSettings::default(), fonts_ready: false },
|
||||
);
|
||||
harness.run();
|
||||
harness
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ use crate::pages::toolbar;
|
||||
use crate::prompt_box::deck::{DeckAction, DeckLimits};
|
||||
use crate::prompt_box::model_selector::model_selector;
|
||||
use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, References};
|
||||
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, editor};
|
||||
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot, editor};
|
||||
|
||||
const PLACEHOLDER: &str = "Describe what you want in the image...";
|
||||
|
||||
@@ -69,38 +69,41 @@ impl CreateImagePage {
|
||||
let deck = if max_images > 0 { DeckMode::References(DeckLimits { max_images, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: env.signed_in }) } else { DeckMode::None };
|
||||
let accepts: &[RefKind] = if max_images > 0 { &[RefKind::Image] } else { &[] };
|
||||
let prompt_max = model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max);
|
||||
let props = PromptBoxProps { id: Id::new("image-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: Some(prompt_max), deck, accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some(), generating: self.generating, generate_tooltip: "Generate", warning: None, banner: None };
|
||||
let props = PromptBoxProps { id: Id::new("image-prompt-box"), placeholder: PLACEHOLDER, enter_to_generate: env.enter_to_generate, mentions: &[], max_length: Some(prompt_max), deck, accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some(), generating: self.generating, generate_tooltip: "Generate", warning: None, banner: None, extra_input: false };
|
||||
let models = &env.catalog.image_page;
|
||||
let settings = &mut self.settings;
|
||||
let mut picked_model = None;
|
||||
let mut left = |ui: &mut Ui| {
|
||||
if let Some(id) = model_selector(ui, Id::new("image-model"), models, model, false) {
|
||||
picked_model = Some(id);
|
||||
}
|
||||
let Some(m) = model else {
|
||||
return;
|
||||
};
|
||||
let aspect = ModelInfo::resolve(settings.aspect_ratio.as_deref(), &m.aspect_ratios, m.aspect_default.as_ref());
|
||||
if let Some(v) = toolbar::aspect_ratio(ui, Id::new("image-aspect"), m, aspect.as_deref().unwrap_or("")) {
|
||||
settings.aspect_ratio = Some(v);
|
||||
}
|
||||
let resolution = ModelInfo::resolve(settings.resolution.as_deref(), &m.resolutions, m.resolution_default.as_ref());
|
||||
if let Some(v) = toolbar::resolution(ui, Id::new("image-resolution"), m, resolution.as_deref().unwrap_or("")) {
|
||||
settings.resolution = Some(v);
|
||||
}
|
||||
let quality = ModelInfo::resolve(settings.quality.as_deref(), &m.qualities, m.quality_default.as_ref());
|
||||
if let Some(v) = toolbar::quality(ui, Id::new("image-quality"), m, quality.as_deref().unwrap_or("")) {
|
||||
settings.quality = Some(v);
|
||||
}
|
||||
};
|
||||
let count = settings.count;
|
||||
let mut picked_model = None;
|
||||
let mut picked_count = None;
|
||||
let mut right = |ui: &mut Ui| {
|
||||
if let Some(m) = model {
|
||||
picked_count = toolbar::count(ui, Id::new("image-count"), m, count, "images");
|
||||
}
|
||||
let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) {
|
||||
(Slot::ToolbarLeft, _) => {
|
||||
if let Some(id) = model_selector(ui, Id::new("image-model"), models, model, false) {
|
||||
picked_model = Some(id);
|
||||
}
|
||||
let Some(m) = model else {
|
||||
return;
|
||||
};
|
||||
let aspect = ModelInfo::resolve(settings.aspect_ratio.as_deref(), &m.aspect_ratios, m.aspect_default.as_ref());
|
||||
if let Some(v) = toolbar::aspect_ratio(ui, Id::new("image-aspect"), m, aspect.as_deref().unwrap_or("")) {
|
||||
settings.aspect_ratio = Some(v);
|
||||
}
|
||||
let resolution = ModelInfo::resolve(settings.resolution.as_deref(), &m.resolutions, m.resolution_default.as_ref());
|
||||
if let Some(v) = toolbar::resolution(ui, Id::new("image-resolution"), m, resolution.as_deref().unwrap_or("")) {
|
||||
settings.resolution = Some(v);
|
||||
}
|
||||
let quality = ModelInfo::resolve(settings.quality.as_deref(), &m.qualities, m.quality_default.as_ref());
|
||||
if let Some(v) = toolbar::quality(ui, Id::new("image-quality"), m, quality.as_deref().unwrap_or("")) {
|
||||
settings.quality = Some(v);
|
||||
}
|
||||
},
|
||||
(Slot::ToolbarRight, Some(m)) => {
|
||||
if let Some(n) = toolbar::count(ui, Id::new("image-count"), m, count, "images") {
|
||||
picked_count = Some(n);
|
||||
}
|
||||
},
|
||||
_ => {},
|
||||
};
|
||||
let actions = prompt_box::show(ui, &mut settings.prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut left, &mut right);
|
||||
let actions = prompt_box::show(ui, &mut settings.prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots);
|
||||
if let Some(id) = picked_model {
|
||||
self.settings.model = Some(id);
|
||||
}
|
||||
@@ -143,6 +146,7 @@ impl CreateImagePage {
|
||||
DeckAction::SwapFrames => {},
|
||||
},
|
||||
PromptBoxAction::DroppedFiles(paths) => self.add_dropped(env, &paths, max_images),
|
||||
PromptBoxAction::MentionPicked(..) => {},
|
||||
PromptBoxAction::PastedImage(png) => {
|
||||
if self.refs.images.len() >= max_images {
|
||||
env.toasts.error(max_images_message(max_images));
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::prompt_box::editor::{self, MentionItem, MentionKind};
|
||||
use crate::prompt_box::model_selector::model_selector;
|
||||
use crate::prompt_box::pickers::{self, PickOption, Trigger};
|
||||
use crate::prompt_box::refs::{ImageSlot, RefKind, RefMedia, References};
|
||||
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState};
|
||||
use crate::prompt_box::{self, DeckMode, PromptBoxAction, PromptBoxProps, PromptBoxState, Slot};
|
||||
use crate::ui::icons::Icon;
|
||||
use crate::ui::widgets::Leading;
|
||||
|
||||
@@ -67,6 +67,8 @@ pub struct CreateVideoPage {
|
||||
pub generating: bool,
|
||||
box_state: PromptBoxState,
|
||||
box_height: f32,
|
||||
/// Which character each mentioned name means, when the user picked one explicitly.
|
||||
character_picks: std::collections::HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl CreateVideoPage {
|
||||
@@ -104,10 +106,14 @@ impl CreateVideoPage {
|
||||
(DeckMode::References(l), _) => [(RefKind::Image, l.max_images), (RefKind::Video, l.max_videos), (RefKind::Audio, l.max_audios)].into_iter().filter(|(_, n)| *n > 0).map(|(k, _)| k).collect(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
let mentions = if reference { self.mentions() } else { Vec::new() };
|
||||
let characters_on = model.is_some_and(supports_characters);
|
||||
let mut mentions = if reference { self.mentions() } else { Vec::new() };
|
||||
if characters_on {
|
||||
mentions.extend(editor::character_mentions(env.characters.iter().map(|c| (c.name.as_str(), c.token.as_str(), c.avatar_url()))));
|
||||
}
|
||||
let needs_start = model.is_some_and(|m| self.needs_start_frame(m));
|
||||
let banner = model.filter(|m| !m.text_to_video && !self.has_image_input()).map(|_| TEXT_ONLY_BANNER);
|
||||
let props = PromptBoxProps { id: Id::new("video-prompt-box"), placeholder: if reference { PLACEHOLDER_REFERENCE } else { PLACEHOLDER }, enter_to_generate: env.enter_to_generate, mentions: &mentions, max_length: Some(model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max)), deck, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !needs_start, generating: self.generating, generate_tooltip: if needs_start { "Add a starting image before generating" } else { "Generate" }, warning: needs_start.then_some("Starting frame required"), banner };
|
||||
let props = PromptBoxProps { id: Id::new("video-prompt-box"), placeholder: if reference { PLACEHOLDER_REFERENCE } else { PLACEHOLDER }, enter_to_generate: env.enter_to_generate, mentions: &mentions, max_length: Some(model.map_or(Some(DEFAULT_PROMPT_MAX), |m| m.prompt_max)), deck, accepts: &accepts, credits: self.cost.credits, generate_enabled: !self.settings.prompt.trim().is_empty() && model.is_some() && !needs_start, generating: self.generating, generate_tooltip: if needs_start { "Add a starting image before generating" } else { "Generate" }, warning: needs_start.then_some("Starting frame required"), banner, extra_input: false };
|
||||
|
||||
let models = &env.catalog.video_page;
|
||||
let mut prompt = std::mem::take(&mut self.settings.prompt);
|
||||
@@ -116,21 +122,28 @@ impl CreateVideoPage {
|
||||
let with_refs = reference && !self.refs.images.is_empty();
|
||||
let mut picked_model = None;
|
||||
let mut picked_mode = None;
|
||||
let mut left = |ui: &mut Ui| {
|
||||
if let Some(id) = model_selector(ui, Id::new("video-model"), models, model, true) {
|
||||
picked_model = Some(id);
|
||||
}
|
||||
if let Some(m) = model {
|
||||
picked_mode = video_toolbar(ui, m, settings, with_refs).or(picked_mode);
|
||||
}
|
||||
};
|
||||
let mut picked_count = None;
|
||||
let mut right = |ui: &mut Ui| {
|
||||
if let Some(m) = model {
|
||||
picked_count = toolbar::count(ui, Id::new("video-count"), m, count, "videos");
|
||||
}
|
||||
let mut open_characters = false;
|
||||
let mut slots = |ui: &mut Ui, slot: Slot| match (slot, model) {
|
||||
(Slot::ToolbarLeft, _) => {
|
||||
if let Some(id) = model_selector(ui, Id::new("video-model"), models, model, true) {
|
||||
picked_model = Some(id);
|
||||
}
|
||||
if let Some(m) = model {
|
||||
picked_mode = video_toolbar(ui, m, settings, with_refs).or(picked_mode);
|
||||
if characters_on && pickers::toggle(ui, Id::new("video-characters"), Icon::User, "@Characters", false, "Characters") {
|
||||
open_characters = true;
|
||||
}
|
||||
}
|
||||
},
|
||||
(Slot::ToolbarRight, Some(m)) => {
|
||||
if let Some(n) = toolbar::count(ui, Id::new("video-count"), m, count, "videos") {
|
||||
picked_count = Some(n);
|
||||
}
|
||||
},
|
||||
_ => {},
|
||||
};
|
||||
let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut left, &mut right);
|
||||
let actions = prompt_box::show(ui, &mut prompt, &self.refs, &mut self.box_state, &props, env.cache, &mut slots);
|
||||
self.settings.prompt = prompt;
|
||||
if let Some(id) = picked_model {
|
||||
self.settings.model = Some(id);
|
||||
@@ -141,6 +154,9 @@ impl CreateVideoPage {
|
||||
if let Some(mode) = picked_mode {
|
||||
switch_mode(&mut self.settings, &mut self.refs, mode);
|
||||
}
|
||||
if open_characters {
|
||||
env.requests.push(AppRequest::OpenCharacters);
|
||||
}
|
||||
for action in actions {
|
||||
self.handle(action, env, model, limits.as_ref());
|
||||
}
|
||||
@@ -176,14 +192,27 @@ impl CreateVideoPage {
|
||||
let item = self.refs.images.remove(from);
|
||||
self.refs.images.insert(to.min(self.refs.images.len()), item);
|
||||
},
|
||||
DeckAction::Preview(id) => {
|
||||
if let Some(url) = self.refs.find_mut(id).and_then(|r| r.full_url.clone().or_else(|| r.preview.clone())) {
|
||||
env.requests.push(AppRequest::Preview(url));
|
||||
}
|
||||
DeckAction::Preview(id) => match self.refs.find_mut(id) {
|
||||
// Audio cards play and stop on click (the webapp's DeckCard).
|
||||
Some(r) if r.kind == RefKind::Audio => {
|
||||
if let Some(url) = r.full_url.clone() {
|
||||
env.requests.push(AppRequest::ToggleAudio { ref_id: id, url });
|
||||
}
|
||||
},
|
||||
Some(r) => {
|
||||
if let Some(url) = r.full_url.clone().or_else(|| r.preview.clone()) {
|
||||
env.requests.push(if r.kind == RefKind::Video { AppRequest::PlayExternally(url) } else { AppRequest::Preview(url) });
|
||||
}
|
||||
},
|
||||
None => {},
|
||||
},
|
||||
DeckAction::SwapFrames => self.refs.swap_frames(),
|
||||
},
|
||||
PromptBoxAction::DroppedFiles(paths) => self.add_files(env, &paths, ImageSlot::Reference, limits),
|
||||
PromptBoxAction::MentionPicked(label, Some(token)) => {
|
||||
self.character_picks.insert(label.trim_start_matches('@').to_owned(), token);
|
||||
},
|
||||
PromptBoxAction::MentionPicked(_, None) => {},
|
||||
PromptBoxAction::PastedImage(png) => match self.settings.input_mode {
|
||||
InputMode::Keyframe => match self.free_keyframe(model) {
|
||||
Some(slot) => common::upload_png(env, &mut self.refs, png, slot),
|
||||
@@ -288,7 +317,27 @@ impl CreateVideoPage {
|
||||
self.generating = true;
|
||||
let ref_image = self.refs.first_frame.as_ref().or(self.refs.images.first()).and_then(|r| r.preview.clone());
|
||||
let meta = EnqueueMeta { prompt: self.settings.prompt.clone(), model_id: model.id.clone(), batch_count: u32::from(model.valid_batch(self.settings.count)), ref_image };
|
||||
env.backend.generate(Modality::Video, self.fields(model, false), meta);
|
||||
let mut fields = self.fields(model, false);
|
||||
if supports_characters(model) {
|
||||
let tokens = mentioned_characters(&self.settings.prompt, env.characters, &self.character_picks);
|
||||
if !tokens.is_empty() {
|
||||
fields.insert("reference_character_tokens".into(), json!(tokens));
|
||||
}
|
||||
}
|
||||
env.backend.generate(Modality::Video, fields, meta);
|
||||
}
|
||||
|
||||
/// The characters dialog picked `name`: mention it at the end of the prompt.
|
||||
pub fn mention_character(&mut self, name: &str, token: &str) {
|
||||
let p = &mut self.settings.prompt;
|
||||
if !p.is_empty() && !p.ends_with(char::is_whitespace) {
|
||||
p.push(' ');
|
||||
}
|
||||
p.push('@');
|
||||
p.push_str(name);
|
||||
p.push(' ');
|
||||
self.character_picks.insert(name.to_owned(), token.to_owned());
|
||||
self.box_state.editor.focus_requested = true;
|
||||
}
|
||||
|
||||
/// The OmniGen request (`GenerateVideoRequest` after `api_fields`).
|
||||
@@ -433,7 +482,7 @@ impl CreateVideoPage {
|
||||
"audioref" => RefKind::Audio,
|
||||
_ => RefKind::Image,
|
||||
};
|
||||
let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256), Some(c.media_links.cdn_url.clone()), 0.0);
|
||||
let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256).filter(|_| kind != RefKind::Audio), Some(c.media_links.cdn_url.clone()), 0.0);
|
||||
match (c.semantic.as_str(), kind) {
|
||||
("vid_end_frame", _) => self.refs.last_frame = Some(item),
|
||||
(_, RefKind::Image) if s.input_mode == InputMode::Keyframe && self.refs.first_frame.is_none() => self.refs.first_frame = Some(item),
|
||||
@@ -506,6 +555,33 @@ fn switch_mode(s: &mut VideoSettings, refs: &mut References, mode: InputMode) {
|
||||
s.input_mode = mode;
|
||||
}
|
||||
|
||||
/// Whether the model takes `@Character` mentions (the listing says so; Seedance 2.0 always does).
|
||||
fn supports_characters(m: &ModelInfo) -> bool {
|
||||
m.character_refs_max > 0 || m.id == "seedance_2p0"
|
||||
}
|
||||
|
||||
/// One token per character named in the prompt (`@Name`, not inside a longer word): the user's
|
||||
/// explicit pick for that name, else the newest character with it.
|
||||
fn mentioned_characters(prompt: &str, characters: &[crate::backend::wire::Character], picks: &std::collections::HashMap<String, String>) -> Vec<String> {
|
||||
let mut names: Vec<&str> = characters.iter().map(|c| c.name.as_str()).filter(|n| !n.is_empty()).collect();
|
||||
names.sort_by_key(|n| std::cmp::Reverse(n.len()));
|
||||
names.dedup();
|
||||
let mut tokens = Vec::new();
|
||||
for name in names {
|
||||
let needle = format!("@{name}");
|
||||
let mentioned = prompt.match_indices(&needle).any(|(i, _)| !prompt[i + needle.len()..].chars().next().is_some_and(|c| c.is_alphanumeric() || c == '_'));
|
||||
if !mentioned {
|
||||
continue;
|
||||
}
|
||||
let candidates: Vec<&crate::backend::wire::Character> = characters.iter().filter(|c| c.name == name).collect();
|
||||
let chosen = picks.get(name).and_then(|t| candidates.iter().find(|c| &c.token == t)).or(candidates.first());
|
||||
if let Some(c) = chosen {
|
||||
tokens.push(c.token.clone());
|
||||
}
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
fn deck_limits(m: &ModelInfo, library: bool) -> DeckLimits {
|
||||
DeckLimits { max_images: m.image_refs_max, max_videos: m.video_refs_max, max_video_secs: m.video_refs_max_secs.map(f32::from), max_audios: m.audio_refs_max, max_audio_secs: m.audio_refs_max_secs.map(f32::from), library }
|
||||
}
|
||||
@@ -539,3 +615,25 @@ fn frames_full_message(model: Option<&ModelInfo>) -> &'static str {
|
||||
"The first frame is already set"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::backend::wire::Character;
|
||||
|
||||
fn character(token: &str, name: &str) -> Character {
|
||||
Character { token: token.into(), name: name.into(), ..Default::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mentions_resolve_to_one_token_per_name() {
|
||||
// Newest first, as the server lists them.
|
||||
let characters = [character("c3", "Bob"), character("c2", "Bob2"), character("c1", "Bob"), character("c0", "Ann")];
|
||||
let none = std::collections::HashMap::new();
|
||||
assert_eq!(mentioned_characters("@Bob2 waves", &characters, &none), ["c2"], "@Bob2 isn't @Bob");
|
||||
assert_eq!(mentioned_characters("@Bob, then @Ann.", &characters, &none), ["c3", "c0"], "newest Bob by default");
|
||||
let picks = std::collections::HashMap::from([("Bob".to_owned(), "c1".to_owned())]);
|
||||
assert_eq!(mentioned_characters("hi @Bob", &characters, &picks), ["c1"], "the explicit pick wins");
|
||||
assert!(mentioned_characters("no mentions", &characters, &none).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod common;
|
||||
pub mod create_audio;
|
||||
pub mod create_image;
|
||||
pub mod create_video;
|
||||
pub mod other;
|
||||
|
||||
@@ -23,12 +23,13 @@ pub fn home(ui: &mut Ui, signed_in: bool, requests: &mut Vec<AppRequest>) -> Opt
|
||||
ui.add_space(10.0);
|
||||
ui.label(widgets::heading("Capable tools for artists.", 52.0));
|
||||
ui.add_space(10.0);
|
||||
ui.label(egui::RichText::new("Generate images and videos with the best models, steer them with references and keyframes, and keep everything in your library.").size(17.0).color(theme::MUTED));
|
||||
ui.label(egui::RichText::new("Generate images, videos and audio with the best models, steer them with references and keyframes, and keep everything in your library.").size(17.0).color(theme::MUTED));
|
||||
ui.add_space(28.0);
|
||||
ui.horizontal(|ui| {
|
||||
ui.spacing_mut().item_spacing.x = 16.0;
|
||||
let card_w = (width - 16.0) / 2.0;
|
||||
for (page, icon, title, text) in [(Page::CreateImage, Icon::Image, "Create Image", "Describe anything. See it in seconds."), (Page::CreateVideo, Icon::Video, "Create Video", "Describe a scene. See it in motion.")] {
|
||||
let cards = [(Page::CreateImage, Icon::Image, "Create Image", "Describe anything. See it in seconds."), (Page::CreateVideo, Icon::Video, "Create Video", "Describe a scene. See it in motion."), (Page::CreateAudio, Icon::Music, "Create Audio", "Describe a song, a sound, or a sample.")];
|
||||
let card_w = (width - 16.0 * (cards.len() - 1) as f32) / cards.len() as f32;
|
||||
for (page, icon, title, text) in cards {
|
||||
if card(ui, card_w, icon, title, text) {
|
||||
open = Some(page);
|
||||
}
|
||||
@@ -73,9 +74,9 @@ pub fn library(ui: &mut Ui, env: &mut Env<'_>, feed: &mut FeedStore) {
|
||||
placeholder(ui, area, "Library", "Your generations will appear here.", None);
|
||||
return;
|
||||
}
|
||||
let view = FeedView { id: Id::new("library-feed"), mode: env.view_mode, pending: &[], failed: &[], items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: true, bottom_padding: 24.0 };
|
||||
let view = FeedView { id: Id::new("library-feed"), mode: env.view_mode, pending: &[], failed: &[], items: &feed.items, has_more: feed.has_more, loading: feed.loading, selecting: feed.selecting, selected: &feed.selected, last_viewed: feed.last_viewed.as_deref(), prompts: env.prompts, make_video: true, autoplay: env.autoplay, bottom_padding: 24.0 };
|
||||
let mut child = ui.new_child(egui::UiBuilder::new().max_rect(area.shrink2(vec2(12.0, 0.0)).with_min_y(area.top() + 2.0)));
|
||||
for action in grid::show(&mut child, &view, env.cache, env.ratios, env.catalog) {
|
||||
for action in grid::show(&mut child, &view, env.cache, env.ratios, env.audio, env.catalog) {
|
||||
match action {
|
||||
FeedAction::Open(token) => {
|
||||
let kind = feed.find(&token).map_or(MediaKind::Image, |i| i.kind);
|
||||
|
||||
@@ -309,12 +309,14 @@ fn paint_card(ui: &Ui, painter: &egui::Painter, rect: Rect, tilt: f32, item: &Re
|
||||
image.paint_at(ui, rect.shrink(2.0));
|
||||
},
|
||||
None => {
|
||||
let playing = item.kind == RefKind::Audio && playing_audio(ui) == Some(item.id);
|
||||
let icon = match item.kind {
|
||||
RefKind::Image => Icon::Image,
|
||||
RefKind::Video => Icon::Video,
|
||||
RefKind::Audio => Icon::Music,
|
||||
RefKind::Audio if playing => Icon::Volume,
|
||||
RefKind::Audio => Icon::Play,
|
||||
};
|
||||
icons::paint(painter, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), icon, theme::MUTED);
|
||||
icons::paint(painter, Rect::from_center_size(rect.center(), vec2(20.0, 20.0)), icon, if playing { theme::ACCENT_INK } else { theme::MUTED });
|
||||
},
|
||||
}
|
||||
if item.kind != RefKind::Image && tilt == 0.0 {
|
||||
@@ -334,6 +336,17 @@ fn paint_card(ui: &Ui, painter: &egui::Painter, rect: Rect, tilt: f32, item: &Re
|
||||
painter.add(Shape::closed_line(corners.to_vec(), Stroke::new(2.0, border)));
|
||||
}
|
||||
|
||||
/// The audio reference playing right now (the app sets it each frame).
|
||||
pub fn set_playing_audio(ctx: &egui::Context, id: Option<u64>) {
|
||||
ctx.data_mut(|d| d.insert_temp(Id::new(PLAYING_AUDIO), id));
|
||||
}
|
||||
|
||||
fn playing_audio(ui: &Ui) -> Option<u64> {
|
||||
ui.data(|d| d.get_temp::<Option<u64>>(Id::new(PLAYING_AUDIO))).flatten()
|
||||
}
|
||||
|
||||
const PLAYING_AUDIO: &str = "deck-playing-audio";
|
||||
|
||||
/// The hover-revealed round × in a card's top-right corner. Returns whether it was clicked.
|
||||
fn remove_button(ui: &mut Ui, id: Id, card: Rect, card_hovered: bool) -> bool {
|
||||
let rect = Rect::from_min_size(pos2(card.right() - 22.0, card.top() + 2.0), vec2(20.0, 20.0));
|
||||
|
||||
@@ -22,12 +22,15 @@ const RIGHT_GUTTER: f32 = 32.0;
|
||||
const IMAGE_COLORS: [Color32; 5] = [Color32::from_rgb(96, 165, 250), Color32::from_rgb(251, 146, 60), Color32::from_rgb(167, 139, 250), Color32::from_rgb(52, 211, 153), Color32::from_rgb(251, 113, 133)];
|
||||
const VIDEO_COLORS: [Color32; 3] = [Color32::from_rgb(250, 204, 21), Color32::from_rgb(245, 158, 11), Color32::from_rgb(74, 222, 128)];
|
||||
const AUDIO_COLORS: [Color32; 2] = [Color32::from_rgb(192, 132, 252), Color32::from_rgb(232, 121, 249)];
|
||||
/// Teal, emerald, sky.
|
||||
const CHARACTER_COLORS: [Color32; 3] = [Color32::from_rgb(45, 212, 191), Color32::from_rgb(34, 197, 94), Color32::from_rgb(14, 165, 233)];
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MentionKind {
|
||||
Image,
|
||||
Video,
|
||||
Audio,
|
||||
Character,
|
||||
}
|
||||
|
||||
/// Something the prompt can reference by name, e.g. `@Image2`.
|
||||
@@ -39,6 +42,8 @@ pub struct MentionItem {
|
||||
pub index: usize,
|
||||
/// Media cache key of a thumbnail, if there is one.
|
||||
pub preview: Option<String>,
|
||||
/// The character this mention names (several can share a name).
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
impl MentionItem {
|
||||
@@ -48,6 +53,7 @@ impl MentionItem {
|
||||
MentionKind::Image => IMAGE_COLORS[i % IMAGE_COLORS.len()],
|
||||
MentionKind::Video => VIDEO_COLORS[i % VIDEO_COLORS.len()],
|
||||
MentionKind::Audio => AUDIO_COLORS[i % AUDIO_COLORS.len()],
|
||||
MentionKind::Character => CHARACTER_COLORS[i % CHARACTER_COLORS.len()],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -93,6 +99,8 @@ pub struct EditorOutput {
|
||||
pub submit: bool,
|
||||
pub open_fullscreen: bool,
|
||||
pub focused: bool,
|
||||
/// A mention the user picked from the dropdown: (label, character token).
|
||||
pub picked: Option<(String, Option<String>)>,
|
||||
}
|
||||
|
||||
/// Draws the editor in the available width.
|
||||
@@ -105,7 +113,7 @@ pub fn show(ui: &mut Ui, prompt: &mut String, state: &mut EditorState, opts: &Ed
|
||||
// Keys the field mustn't see: dropdown navigation, and Enter when it generates.
|
||||
if had_focus {
|
||||
if !filtered.is_empty() {
|
||||
handle_dropdown_keys(ui, id, prompt, state, &filtered);
|
||||
out.picked = handle_dropdown_keys(ui, id, prompt, state, &filtered);
|
||||
} else if opts.enter_to_generate && consume_plain_enter(ui) {
|
||||
out.submit = true;
|
||||
}
|
||||
@@ -183,7 +191,9 @@ pub fn show(ui: &mut Ui, prompt: &mut String, state: &mut EditorState, opts: &Ed
|
||||
}
|
||||
|
||||
if !filtered.is_empty() && out.focused {
|
||||
mention_dropdown(ui, id, field_rect, prompt, state, &filtered, thumbnails);
|
||||
if let Some(picked) = mention_dropdown(ui, id, field_rect, prompt, state, &filtered, thumbnails) {
|
||||
out.picked = Some(picked);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -214,12 +224,10 @@ fn consume_plain_enter(ui: &Ui) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem]) {
|
||||
fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem]) -> Option<(String, Option<String>)> {
|
||||
let n = filtered.len();
|
||||
let (down, up, accept, escape) = ui.input_mut(|i| (i.consume_key(Modifiers::NONE, Key::ArrowDown), i.consume_key(Modifiers::NONE, Key::ArrowUp), i.consume_key(Modifiers::NONE, Key::Enter) || i.consume_key(Modifiers::NONE, Key::Tab), i.consume_key(Modifiers::NONE, Key::Escape)));
|
||||
let Some(query) = state.mention.as_mut() else {
|
||||
return;
|
||||
};
|
||||
let query = state.mention.as_mut()?;
|
||||
if down {
|
||||
query.selected = (query.selected + 1) % n;
|
||||
}
|
||||
@@ -228,12 +236,14 @@ fn handle_dropdown_keys(ui: &Ui, id: Id, prompt: &mut String, state: &mut Editor
|
||||
}
|
||||
if escape {
|
||||
state.mention = None;
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
if accept {
|
||||
let label = filtered[query.selected.min(n - 1)].label.clone();
|
||||
insert_mention(ui.ctx(), id, prompt, state, &label);
|
||||
if !accept {
|
||||
return None;
|
||||
}
|
||||
let item = filtered[query.selected.min(n - 1)];
|
||||
insert_mention(ui.ctx(), id, prompt, state, &item.label);
|
||||
Some((item.label.clone(), item.token.clone()))
|
||||
}
|
||||
|
||||
/// Opens, updates or closes the dropdown from the text before the cursor.
|
||||
@@ -284,7 +294,7 @@ fn insert_mention(ctx: &egui::Context, id: Id, prompt: &mut String, state: &mut
|
||||
}
|
||||
}
|
||||
|
||||
fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem], thumbnails: &mut dyn FnMut(&Ui, &str) -> Option<egui::TextureHandle>) {
|
||||
fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &mut EditorState, filtered: &[&MentionItem], thumbnails: &mut dyn FnMut(&Ui, &str) -> Option<egui::TextureHandle>) -> Option<(String, Option<String>)> {
|
||||
let selected = state.mention.as_ref().map_or(0, |q| q.selected);
|
||||
let row_h = 44.0;
|
||||
let h = 28.0 + row_h * filtered.len() as f32;
|
||||
@@ -317,23 +327,25 @@ fn mention_dropdown(ui: &Ui, id: Id, field: Rect, prompt: &mut String, state: &m
|
||||
MentionKind::Image => Icon::Image,
|
||||
MentionKind::Video => Icon::Video,
|
||||
MentionKind::Audio => Icon::Music,
|
||||
MentionKind::Character => Icon::User,
|
||||
};
|
||||
icons::paint(ui.painter(), tile.shrink(9.0), icon, theme::fade(Color32::WHITE, 0.6));
|
||||
},
|
||||
}
|
||||
ui.painter().text(pos2(tile.right() + 10.0, row.center().y), Align2::LEFT_CENTER, &item.label, FontId::new(14.0, theme::medium()), item.color());
|
||||
if resp.clicked() {
|
||||
picked = Some(item.label.clone());
|
||||
picked = Some((item.label.clone(), item.token.clone()));
|
||||
}
|
||||
}
|
||||
});
|
||||
if let (Some(i), Some(q)) = (hovered, state.mention.as_mut()) {
|
||||
q.selected = i;
|
||||
}
|
||||
if let Some(label) = picked {
|
||||
insert_mention(ui.ctx(), id, prompt, state, &label);
|
||||
if let Some((label, _)) = &picked {
|
||||
insert_mention(ui.ctx(), id, prompt, state, label);
|
||||
ui.memory_mut(|m| m.request_focus(id));
|
||||
}
|
||||
picked
|
||||
}
|
||||
|
||||
/// The three-line grip that resizes the field (`.promptbox-resize-wrap::after`).
|
||||
@@ -382,14 +394,20 @@ fn highlight(text: &str, mentions: &[MentionItem]) -> LayoutJob {
|
||||
job
|
||||
}
|
||||
|
||||
/// `@Name` mentions for characters (newest first), coloured by position.
|
||||
pub fn character_mentions<'a>(characters: impl Iterator<Item = (&'a str, &'a str, Option<String>)>) -> Vec<MentionItem> {
|
||||
characters.enumerate().map(|(i, (name, token, avatar))| MentionItem { label: format!("@{name}"), kind: MentionKind::Character, index: i + 1, preview: avatar, token: Some(token.to_owned()) }).collect()
|
||||
}
|
||||
|
||||
/// Mention labels for `n` items of `kind` (`@Image1`, `@Image2`, …).
|
||||
pub fn mention_items(kind: MentionKind, previews: impl Iterator<Item = Option<String>>) -> Vec<MentionItem> {
|
||||
let name = match kind {
|
||||
MentionKind::Image => "Image",
|
||||
MentionKind::Video => "Video",
|
||||
MentionKind::Audio => "Audio",
|
||||
MentionKind::Character => "Character",
|
||||
};
|
||||
previews.enumerate().map(|(i, preview)| MentionItem { label: format!("@{name}{}", i + 1), kind, index: i + 1, preview }).collect()
|
||||
previews.enumerate().map(|(i, preview)| MentionItem { label: format!("@{name}{}", i + 1), kind, index: i + 1, preview, token: None }).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -58,6 +58,21 @@ pub struct PromptBoxProps<'a> {
|
||||
pub warning: Option<&'a str>,
|
||||
/// An info strip above the prompt.
|
||||
pub banner: Option<&'a str>,
|
||||
/// Something clear-all also clears besides the prompt and references (the audio style).
|
||||
pub extra_input: bool,
|
||||
}
|
||||
|
||||
/// Where a page draws into the box.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Slot {
|
||||
/// Rows above the prompt (the audio box's references).
|
||||
AbovePrompt,
|
||||
/// Rows below the prompt (the audio box's style).
|
||||
BelowPrompt,
|
||||
/// The toolbar's left: the model selector and pickers.
|
||||
ToolbarLeft,
|
||||
/// The toolbar's right, before clear-all: the count picker.
|
||||
ToolbarRight,
|
||||
}
|
||||
|
||||
/// Something the page should do.
|
||||
@@ -68,6 +83,8 @@ pub enum PromptBoxAction {
|
||||
ClearAll,
|
||||
DroppedFiles(Vec<PathBuf>),
|
||||
PastedImage(Vec<u8>),
|
||||
/// A mention picked from the dropdown: (label, character token).
|
||||
MentionPicked(String, Option<String>),
|
||||
}
|
||||
|
||||
/// UI state that outlives a frame.
|
||||
@@ -79,11 +96,10 @@ pub struct PromptBoxState {
|
||||
focused: bool,
|
||||
}
|
||||
|
||||
/// Draws the box in `ui` (which should be as wide as the box). `toolbar_left` holds the model
|
||||
/// selector and pickers; `toolbar_right` the count picker. Both may be drawn twice in a frame
|
||||
/// (box and focus mode), each under its own id.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, toolbar_left: &mut dyn FnMut(&mut Ui), toolbar_right: &mut dyn FnMut(&mut Ui)) -> Vec<PromptBoxAction> {
|
||||
/// Draws the box in `ui` (which should be as wide as the box). `slots` draws the page's parts
|
||||
/// into each [`Slot`]; the toolbar's left may be drawn twice in a frame (box and focus mode), each
|
||||
/// under its own id.
|
||||
pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, slots: &mut dyn FnMut(&mut Ui, Slot)) -> Vec<PromptBoxAction> {
|
||||
let mut actions = Vec::new();
|
||||
let frame = egui::Frame::new().fill(theme::fade(theme::CONTROLS, 0.95)).stroke(Stroke::new(1.0, if state.focused { theme::ACCENT } else { theme::LINE })).corner_radius(theme::RADIUS).inner_margin(16);
|
||||
let response = frame.show(ui, |ui| {
|
||||
@@ -94,25 +110,30 @@ pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut Pro
|
||||
ui.spacing_mut().item_spacing.x = 12.0;
|
||||
deck_widget(ui, props, refs, cache, false, &mut actions);
|
||||
ui.vertical(|ui| {
|
||||
slots(ui, Slot::AbovePrompt);
|
||||
let out = editor_ui(ui, prompt, state, props, cache, None);
|
||||
state.focused = out.focused;
|
||||
if out.submit {
|
||||
actions.push(PromptBoxAction::Generate);
|
||||
}
|
||||
if let Some((label, token)) = out.picked {
|
||||
actions.push(PromptBoxAction::MentionPicked(label, token));
|
||||
}
|
||||
if out.open_fullscreen {
|
||||
state.fullscreen = true;
|
||||
}
|
||||
slots(ui, Slot::BelowPrompt);
|
||||
});
|
||||
});
|
||||
ui.add_space(14.0);
|
||||
ui.horizontal(|ui| {
|
||||
ui.spacing_mut().item_spacing.x = 8.0;
|
||||
toolbar_left(ui);
|
||||
slots(ui, Slot::ToolbarLeft);
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
ui.spacing_mut().item_spacing.x = 6.0;
|
||||
generate_cluster(ui, props, &mut actions);
|
||||
clear_all_button(ui, prompt, refs, state, &mut actions);
|
||||
toolbar_right(ui);
|
||||
clear_all_button(ui, prompt, refs, state, props.extra_input, &mut actions);
|
||||
slots(ui, Slot::ToolbarRight);
|
||||
if let Some(warning) = props.warning {
|
||||
warning_label(ui, warning);
|
||||
}
|
||||
@@ -124,7 +145,7 @@ pub fn show(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut Pro
|
||||
drop_target(ui, props, rect, &mut actions);
|
||||
paste_images(ui, props, &mut actions);
|
||||
if state.fullscreen {
|
||||
focus_mode(ui.ctx(), prompt, refs, state, props, cache, toolbar_left, &mut actions);
|
||||
focus_mode(ui.ctx(), prompt, refs, state, props, cache, slots, &mut actions);
|
||||
}
|
||||
if state.confirm_clear {
|
||||
confirm_clear(ui.ctx(), props.id, state, &mut actions);
|
||||
@@ -167,10 +188,10 @@ fn generate_cluster(ui: &mut Ui, props: &PromptBoxProps<'_>, actions: &mut Vec<P
|
||||
}
|
||||
|
||||
/// The eraser button and the hairline after it (`PromptClearAllButton`).
|
||||
fn clear_all_button(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, actions: &mut Vec<PromptBoxAction>) {
|
||||
fn clear_all_button(ui: &mut Ui, prompt: &mut String, refs: &References, state: &mut PromptBoxState, extra_input: bool, actions: &mut Vec<PromptBoxAction>) {
|
||||
let (rule, _) = ui.allocate_exact_size(vec2(1.0, 20.0), Sense::hover());
|
||||
ui.painter().rect_filled(rule, 0.0, theme::fade(theme::INK, 0.15));
|
||||
let clearable = !prompt.is_empty() || !refs.is_empty();
|
||||
let clearable = !prompt.is_empty() || !refs.is_empty() || extra_input;
|
||||
let resp = ui.add_enabled_ui(clearable, |ui| widgets::ghost(ui, Icon::Eraser, None, "Clear all", false)).inner;
|
||||
if resp.clicked() {
|
||||
if refs.is_empty() {
|
||||
@@ -307,7 +328,7 @@ fn paste_images(ui: &mut Ui, props: &PromptBoxProps<'_>, actions: &mut Vec<Promp
|
||||
|
||||
/// Focus mode: a large editor in a modal, with the deck and toolbar at hand.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, toolbar_left: &mut dyn FnMut(&mut Ui), actions: &mut Vec<PromptBoxAction>) {
|
||||
fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state: &mut PromptBoxState, props: &PromptBoxProps<'_>, cache: &mut MediaCache, slots: &mut dyn FnMut(&mut Ui, Slot), actions: &mut Vec<PromptBoxAction>) {
|
||||
let height = ctx.content_rect().height() * 0.7;
|
||||
let mut done = false;
|
||||
let closed = widgets::modal(ctx, props.id.with("focus-mode"), 896.0, |ui| {
|
||||
@@ -330,12 +351,12 @@ fn focus_mode(ctx: &egui::Context, prompt: &mut String, refs: &References, state
|
||||
ui.push_id("focus-deck", |ui| deck_widget(ui, props, refs, cache, true, actions));
|
||||
ui.add_space(12.0);
|
||||
ui.horizontal(|ui| {
|
||||
ui.push_id("focus-toolbar", |ui| ui.horizontal(|ui| toolbar_left(ui)));
|
||||
ui.push_id("focus-toolbar", |ui| ui.horizontal(|ui| slots(ui, Slot::ToolbarLeft)));
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
if widgets::button(ui, None, "Done", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() {
|
||||
done = true;
|
||||
}
|
||||
ui.push_id("focus-clear", |ui| clear_all_button(ui, prompt, refs, state, actions));
|
||||
ui.push_id("focus-clear", |ui| clear_all_button(ui, prompt, refs, state, props.extra_input, actions));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -90,14 +90,19 @@ pub fn popover(anchor: &Response, popup_id: Id, content: impl FnOnce(&mut Ui)) {
|
||||
});
|
||||
}
|
||||
|
||||
/// A toolbar pill that opens a popover of custom `content` above it.
|
||||
pub fn popover_pill(ui: &mut Ui, id: Id, lead: Leading<'_>, label: &str, tooltip: &str, content: impl FnOnce(&mut Ui)) {
|
||||
let popup_id = ui.make_persistent_id(id).with("popup");
|
||||
let open = Popup::is_id_open(ui.ctx(), popup_id);
|
||||
let resp = ui.push_id(id, |ui| widgets::pill(ui, lead, label, open)).inner.on_hover_text(tooltip);
|
||||
popover(&resp, popup_id, content);
|
||||
}
|
||||
|
||||
/// The duration popover: a clock pill reading `5s`, opening a 1-second-step slider between the
|
||||
/// model's minimum and maximum. Returns the new value.
|
||||
pub fn duration_picker(ui: &mut Ui, id: Id, current: u32, min: u32, max: u32) -> Option<u32> {
|
||||
let popup_id = ui.make_persistent_id(id).with("popup");
|
||||
let open = Popup::is_id_open(ui.ctx(), popup_id);
|
||||
let resp = ui.push_id(id, |ui| widgets::pill(ui, Leading::Icon(Icon::Clock), &format!("{current}s"), open)).inner.on_hover_text("Duration");
|
||||
let mut value = current;
|
||||
popover(&resp, popup_id, |ui| {
|
||||
popover_pill(ui, id, Leading::Icon(Icon::Clock), &format!("{current}s"), "Duration", |ui| {
|
||||
ui.set_width(240.0);
|
||||
widgets::menu_header(ui, "Duration");
|
||||
ui.add_space(4.0);
|
||||
|
||||
@@ -11,7 +11,7 @@ use crate::pages::toolbar;
|
||||
use crate::prompt_box::deck::DeckLimits;
|
||||
use crate::prompt_box::editor::{MentionKind, mention_items};
|
||||
use crate::prompt_box::refs::{RefKind, References};
|
||||
use crate::prompt_box::{DeckMode, PromptBoxProps, PromptBoxState, show};
|
||||
use crate::prompt_box::{DeckMode, PromptBoxProps, PromptBoxState, Slot, show};
|
||||
|
||||
struct State {
|
||||
// Keeps the media cache's runtime alive.
|
||||
@@ -45,15 +45,16 @@ fn build(state: State) -> Harness<'static, State> {
|
||||
return;
|
||||
}
|
||||
let mentions = if s.with_mentions { mention_items(MentionKind::Image, [None, None].into_iter()) } else { Vec::new() };
|
||||
let props = PromptBoxProps { id: egui::Id::new("test-box"), placeholder: "Describe what you want in the image...", enter_to_generate: s.enter_to_generate, mentions: &mentions, max_length: Some(Some(1000)), deck: DeckMode::References(DeckLimits { max_images: 4, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: false }), accepts: &[RefKind::Image], credits: Some(15), generate_enabled: !s.prompt.trim().is_empty(), generating: false, generate_tooltip: "Generate", warning: None, banner: None };
|
||||
let props = PromptBoxProps { id: egui::Id::new("test-box"), placeholder: "Describe what you want in the image...", enter_to_generate: s.enter_to_generate, mentions: &mentions, max_length: Some(Some(1000)), deck: DeckMode::References(DeckLimits { max_images: 4, max_videos: 0, max_video_secs: None, max_audios: 0, max_audio_secs: None, library: false }), accepts: &[RefKind::Image], credits: Some(15), generate_enabled: !s.prompt.trim().is_empty(), generating: false, generate_tooltip: "Generate", warning: None, banner: None, extra_input: false };
|
||||
let (model, aspect) = (&s.model, &mut s.aspect);
|
||||
let mut left = |ui: &mut egui::Ui| {
|
||||
if let Some(v) = toolbar::aspect_ratio(ui, egui::Id::new("test-aspect"), model, aspect) {
|
||||
*aspect = v;
|
||||
let mut slots = |ui: &mut egui::Ui, slot: Slot| {
|
||||
if slot == Slot::ToolbarLeft {
|
||||
if let Some(v) = toolbar::aspect_ratio(ui, egui::Id::new("test-aspect"), model, aspect) {
|
||||
*aspect = v;
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut right = |_: &mut egui::Ui| {};
|
||||
let actions = show(ui, &mut s.prompt, &s.refs, &mut s.box_state, &props, &mut s.cache, &mut left, &mut right);
|
||||
let actions = show(ui, &mut s.prompt, &s.refs, &mut s.box_state, &props, &mut s.cache, &mut slots);
|
||||
s.actions.extend(actions.iter().map(|a| format!("{a:?}")));
|
||||
},
|
||||
state,
|
||||
@@ -128,7 +129,7 @@ fn typing_at_offers_mentions_and_enter_inserts_one() {
|
||||
harness.key_press(egui::Key::Enter);
|
||||
harness.run();
|
||||
assert_eq!(harness.state().prompt, "put @Image2 ");
|
||||
assert!(harness.state().actions.is_empty(), "Enter picked a mention, it didn't generate");
|
||||
assert_eq!(harness.state().actions, [r#"MentionPicked("@Image2", None)"#], "Enter picked a mention, it didn't generate");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -87,6 +87,7 @@ impl Page {
|
||||
"home" => Page::Home,
|
||||
"create-image" | "image" => Page::CreateImage,
|
||||
"create-video" | "video" => Page::CreateVideo,
|
||||
"create-audio" | "audio" => Page::CreateAudio,
|
||||
"library" => Page::Library,
|
||||
_ => return None,
|
||||
})
|
||||
@@ -94,7 +95,7 @@ impl Page {
|
||||
|
||||
/// Pages the native app has; the rest open a "coming soon" page.
|
||||
pub fn is_ready(self) -> bool {
|
||||
matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::Library)
|
||||
matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::CreateAudio | Page::Library)
|
||||
}
|
||||
|
||||
/// The same tool in the web app.
|
||||
@@ -133,6 +134,7 @@ pub enum ShellAction {
|
||||
SignOut,
|
||||
OpenSettings,
|
||||
SetViewMode(ViewMode),
|
||||
SetAutoplay(bool),
|
||||
ToggleSelect,
|
||||
}
|
||||
|
||||
@@ -286,8 +288,17 @@ fn account_row(ui: &mut Ui, name: &str, credits: Option<u64>) -> egui::Response
|
||||
resp.on_hover_cursor(egui::CursorIcon::PointingHand).on_hover_text("Settings")
|
||||
}
|
||||
|
||||
/// The content panel's header: breadcrumbs, and the select and grid/list toggles on feed pages.
|
||||
pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option<(ViewMode, bool)>) -> Option<ShellAction> {
|
||||
/// The feed toggles a page's header shows.
|
||||
pub struct FeedToggles {
|
||||
pub mode: ViewMode,
|
||||
pub selecting: bool,
|
||||
/// Shown on pages with videos (`GalleryAutoplayToggle`).
|
||||
pub autoplay: Option<bool>,
|
||||
}
|
||||
|
||||
/// The content panel's header: breadcrumbs, and the select, previews and grid/list toggles on
|
||||
/// feed pages.
|
||||
pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option<FeedToggles>) -> Option<ShellAction> {
|
||||
let mut action = None;
|
||||
let rect = ui.max_rect();
|
||||
ui.painter().hline(rect.x_range(), rect.bottom() - 0.5, theme::hairline());
|
||||
@@ -302,12 +313,18 @@ pub fn page_header(ui: &mut Ui, page: Page, feed_toggles: Option<(ViewMode, bool
|
||||
let (crumb, _) = ui.allocate_exact_size(galley.size() + vec2(12.0, 8.0), Sense::hover());
|
||||
ui.painter().rect_filled(crumb, 0.0, theme::INK);
|
||||
ui.painter().galley(crumb.center() - galley.size() / 2.0, galley, theme::BG);
|
||||
if let Some((mode, selecting)) = feed_toggles {
|
||||
if let Some(t) = feed_toggles {
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
if let Some(m) = view_toggle(ui, mode) {
|
||||
if let Some(m) = view_toggle(ui, t.mode) {
|
||||
action = Some(ShellAction::SetViewMode(m));
|
||||
}
|
||||
if segmented(ui, &[(Icon::SquareCheck, if selecting { "Exit selection" } else { "Select items" }, selecting)]).is_some() {
|
||||
if let Some(playing) = t.autoplay {
|
||||
// Right-to-left: stills first so "play" ends up on the left.
|
||||
if let Some(i) = segmented(ui, &[(Icon::Image, "Still thumbnails", !playing), (Icon::Play, "Play video previews", playing)]) {
|
||||
action = Some(ShellAction::SetAutoplay(i == 1));
|
||||
}
|
||||
}
|
||||
if segmented(ui, &[(Icon::SquareCheck, if t.selecting { "Exit selection" } else { "Select items" }, t.selecting)]).is_some() {
|
||||
action = Some(ShellAction::ToggleSelect);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@ pub enum Icon {
|
||||
Coins,
|
||||
Copy,
|
||||
Download,
|
||||
Drum,
|
||||
Eraser,
|
||||
Eye,
|
||||
ExternalLink,
|
||||
@@ -35,17 +36,24 @@ pub enum Icon {
|
||||
LogIn,
|
||||
LogOut,
|
||||
Maximize,
|
||||
Mic,
|
||||
MicOff,
|
||||
Music,
|
||||
PanelLeft,
|
||||
Pause,
|
||||
Pencil,
|
||||
Play,
|
||||
Plus,
|
||||
Repeat,
|
||||
RotateCw,
|
||||
Settings,
|
||||
SlidersHorizontal,
|
||||
Sparkles,
|
||||
SquareCheck,
|
||||
Tags,
|
||||
Trash,
|
||||
Upload,
|
||||
User,
|
||||
Video,
|
||||
Volume,
|
||||
VolumeOff,
|
||||
@@ -157,6 +165,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
g.line(p, &[(7.0, 10.0), (12.0, 15.0), (17.0, 10.0)]);
|
||||
g.line(p, &[(4.0, 21.0), (20.0, 21.0)]);
|
||||
},
|
||||
Icon::Drum => {
|
||||
g.line(p, &[(2.0, 2.0), (10.0, 8.0)]);
|
||||
g.line(p, &[(22.0, 2.0), (14.0, 8.0)]);
|
||||
g.closed(p, &[(2.0, 9.0), (6.0, 6.5), (12.0, 5.5), (18.0, 6.5), (22.0, 9.0), (18.0, 11.5), (12.0, 12.5), (6.0, 11.5)]);
|
||||
g.line(p, &[(2.0, 9.0), (2.0, 17.0), (6.0, 19.5), (12.0, 20.5), (18.0, 19.5), (22.0, 17.0), (22.0, 9.0)]);
|
||||
g.line(p, &[(7.0, 11.8), (7.0, 19.8)]);
|
||||
g.line(p, &[(17.0, 11.8), (17.0, 19.8)]);
|
||||
},
|
||||
Icon::Eraser => {
|
||||
g.closed(p, &[(7.0, 21.0), (3.5, 17.5), (14.0, 7.0), (21.0, 14.0), (14.0, 21.0)]);
|
||||
g.line(p, &[(22.0, 21.0), (7.0, 21.0)]);
|
||||
@@ -248,6 +264,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
g.line(p, &[(21.0, 3.0), (14.0, 10.0)]);
|
||||
g.line(p, &[(3.0, 21.0), (10.0, 14.0)]);
|
||||
},
|
||||
Icon::Mic | Icon::MicOff => {
|
||||
g.rect(p, 9.0, 2.0, 6.0, 13.0, 3.0);
|
||||
g.line(p, &[(19.0, 10.0), (19.0, 12.0), (17.0, 16.0), (12.0, 19.0), (7.0, 16.0), (5.0, 12.0), (5.0, 10.0)]);
|
||||
g.line(p, &[(12.0, 19.0), (12.0, 22.0)]);
|
||||
if icon == Icon::MicOff {
|
||||
g.line(p, &[(2.0, 2.0), (22.0, 22.0)]);
|
||||
}
|
||||
},
|
||||
Icon::Music => {
|
||||
g.line(p, &[(9.0, 18.0), (9.0, 5.0), (21.0, 3.0), (21.0, 16.0)]);
|
||||
g.circle(p, 6.0, 18.0, 3.0);
|
||||
@@ -257,6 +281,10 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
g.rect(p, 3.0, 3.0, 18.0, 18.0, 2.0);
|
||||
g.line(p, &[(9.0, 3.0), (9.0, 21.0)]);
|
||||
},
|
||||
Icon::Pause => {
|
||||
p.rect_filled(Rect::from_min_size(g.p(6.0, 4.0), vec2(4.0, 16.0) * g.scale), 1.0, color);
|
||||
p.rect_filled(Rect::from_min_size(g.p(14.0, 4.0), vec2(4.0, 16.0) * g.scale), 1.0, color);
|
||||
},
|
||||
Icon::Pencil => {
|
||||
g.closed(p, &[(17.0, 3.0), (21.0, 7.0), (7.5, 20.5), (2.0, 22.0), (3.5, 16.5)]);
|
||||
g.line(p, &[(15.0, 5.0), (19.0, 9.0)]);
|
||||
@@ -269,6 +297,12 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
g.line(p, &[(5.0, 12.0), (19.0, 12.0)]);
|
||||
g.line(p, &[(12.0, 5.0), (12.0, 19.0)]);
|
||||
},
|
||||
Icon::Repeat => {
|
||||
g.line(p, &[(17.0, 2.0), (21.0, 6.0), (17.0, 10.0)]);
|
||||
g.line(p, &[(3.0, 11.0), (3.0, 10.0), (5.0, 7.0), (7.0, 6.0), (21.0, 6.0)]);
|
||||
g.line(p, &[(7.0, 22.0), (3.0, 18.0), (7.0, 14.0)]);
|
||||
g.line(p, &[(21.0, 13.0), (21.0, 14.0), (19.0, 17.0), (17.0, 18.0), (3.0, 18.0)]);
|
||||
},
|
||||
Icon::RotateCw => {
|
||||
g.arc(p, 12.0, 12.0, 9.0, -60.0, 250.0);
|
||||
g.line(p, &[(21.0, 3.0), (21.0, 8.5), (15.5, 8.5)]);
|
||||
@@ -281,6 +315,17 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
g.line(p, &[(12.0 + 7.5 * a.cos(), 12.0 + 7.5 * a.sin()), (12.0 + 10.0 * a.cos(), 12.0 + 10.0 * a.sin())]);
|
||||
}
|
||||
},
|
||||
Icon::SlidersHorizontal => {
|
||||
g.line(p, &[(21.0, 4.0), (14.0, 4.0)]);
|
||||
g.line(p, &[(10.0, 4.0), (3.0, 4.0)]);
|
||||
g.line(p, &[(21.0, 12.0), (12.0, 12.0)]);
|
||||
g.line(p, &[(8.0, 12.0), (3.0, 12.0)]);
|
||||
g.line(p, &[(21.0, 20.0), (16.0, 20.0)]);
|
||||
g.line(p, &[(12.0, 20.0), (3.0, 20.0)]);
|
||||
g.line(p, &[(14.0, 2.0), (14.0, 6.0)]);
|
||||
g.line(p, &[(8.0, 10.0), (8.0, 14.0)]);
|
||||
g.line(p, &[(16.0, 18.0), (16.0, 22.0)]);
|
||||
},
|
||||
Icon::Sparkles => {
|
||||
let star = |cx: f32, cy: f32, r: f32| {
|
||||
let k = r * 0.28;
|
||||
@@ -289,6 +334,11 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
star(10.0, 12.0, 8.0);
|
||||
star(19.0, 5.0, 3.0);
|
||||
},
|
||||
Icon::Tags => {
|
||||
g.closed(p, &[(3.0, 5.0), (5.0, 3.0), (10.6, 3.0), (18.0, 10.4), (18.0, 12.6), (12.6, 18.0), (10.4, 18.0), (3.0, 10.6)]);
|
||||
g.line(p, &[(15.0, 3.0), (22.0, 10.0), (22.0, 12.0), (17.0, 17.0)]);
|
||||
g.dot(p, 7.5, 7.5, 1.1);
|
||||
},
|
||||
Icon::Trash => {
|
||||
g.line(p, &[(3.0, 6.0), (21.0, 6.0)]);
|
||||
g.line(p, &[(19.0, 6.0), (19.0, 20.0), (17.0, 22.0), (7.0, 22.0), (5.0, 20.0), (5.0, 6.0)]);
|
||||
@@ -299,6 +349,10 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
||||
g.line(p, &[(7.0, 8.0), (12.0, 3.0), (17.0, 8.0)]);
|
||||
g.line(p, &[(4.0, 21.0), (20.0, 21.0)]);
|
||||
},
|
||||
Icon::User => {
|
||||
g.circle(p, 12.0, 8.0, 4.5);
|
||||
g.arc(p, 12.0, 22.0, 8.0, 180.0, 360.0);
|
||||
},
|
||||
Icon::Video => {
|
||||
g.rect(p, 2.0, 6.0, 14.0, 12.0, 2.0);
|
||||
g.closed(p, &[(16.0, 10.5), (22.0, 7.0), (22.0, 17.0), (16.0, 13.5)]);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
pub mod creator_icons;
|
||||
pub mod icons;
|
||||
pub mod toast;
|
||||
pub mod waveform;
|
||||
pub mod widgets;
|
||||
pub mod window;
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
//! The track player (`WaveformAudioPlayer`): a round white play button, the waveform (click to
|
||||
//! seek, played part in blue), the time, and a volume flyout. Placeholder bars stand in while the
|
||||
//! waveform decodes; a plain progress line if it can't.
|
||||
|
||||
use egui::{Color32, FontId, Id, Popup, Rect, Sense, Stroke, Ui, pos2, vec2};
|
||||
|
||||
use crate::backend::audio::{self, AudioPlayer, WaveformLookup};
|
||||
use crate::prompt_box::pickers;
|
||||
use crate::theme;
|
||||
use crate::ui::icons::{self, Icon};
|
||||
|
||||
/// Bars drawn while the waveform loads (`PLACEHOLDER_BAR_COUNT`).
|
||||
const PLACEHOLDER_BARS: usize = 48;
|
||||
/// Waveform bars: width and the step between them.
|
||||
const BAR_WIDTH: f32 = 2.0;
|
||||
const BAR_STEP: f32 = 3.5;
|
||||
const WAVE_INK: Color32 = Color32::from_rgba_premultiplied(89, 89, 89, 89);
|
||||
const PROGRESS_INK: Color32 = Color32::from_rgb(0x2d, 0x81, 0xff);
|
||||
/// How often a playing track redraws its progress.
|
||||
const FRAME: std::time::Duration = std::time::Duration::from_millis(33);
|
||||
|
||||
/// Draws the player across `ui`'s width; `duration_hint` (seconds) is used until the waveform
|
||||
/// knows the real length. `compact` is the list rows' smaller size. Returns the player's rect.
|
||||
pub fn player(ui: &mut Ui, id: Id, audio: &mut AudioPlayer, url: &str, duration_hint: Option<f32>, compact: bool) -> Rect {
|
||||
let (button_d, wave_h): (f32, f32) = if compact { (32.0, 32.0) } else { (40.0, 44.0) };
|
||||
let (rect, _) = ui.allocate_exact_size(vec2(ui.available_width(), wave_h.max(button_d)), Sense::hover());
|
||||
// Clicks between the controls stay with the player (they don't open the card underneath).
|
||||
ui.interact(rect, id.with("bg"), Sense::click());
|
||||
let ctx = ui.ctx().clone();
|
||||
let waveform = audio.waveform(&ctx, url);
|
||||
let playback = audio.track(url);
|
||||
let duration = match &waveform {
|
||||
WaveformLookup::Ready(w) => Some(w.duration),
|
||||
_ => duration_hint,
|
||||
}
|
||||
.filter(|d| *d > 0.0);
|
||||
let position = playback.map_or(0.0, |p| p.position);
|
||||
let playing = playback.is_some_and(|p| !p.paused && !p.loading);
|
||||
if playing {
|
||||
ctx.request_repaint_after(FRAME);
|
||||
}
|
||||
|
||||
// Play / pause.
|
||||
let button = Rect::from_center_size(pos2(rect.left() + button_d / 2.0, rect.center().y), vec2(button_d, button_d));
|
||||
let resp = ui.interact(button, id.with("play"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
let label = if playing { "Pause" } else { "Play" };
|
||||
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, label));
|
||||
ui.painter().circle_filled(button.center(), button_d / 2.0, if resp.hovered() { theme::fade(Color32::WHITE, 0.88) } else { Color32::WHITE });
|
||||
let glyph = Rect::from_center_size(button.center(), vec2(button_d * 0.4, button_d * 0.4));
|
||||
match playback {
|
||||
Some(p) if p.loading => icons::paint_spinner(ui, glyph, Color32::BLACK),
|
||||
_ if playing => icons::paint(ui.painter(), glyph, Icon::Pause, Color32::BLACK),
|
||||
_ => icons::paint(ui.painter(), glyph.translate(vec2(button_d * 0.04, 0.0)), Icon::Play, Color32::BLACK),
|
||||
}
|
||||
if resp.clicked() {
|
||||
audio.toggle_track(&ctx, url);
|
||||
}
|
||||
|
||||
// Volume, then the time, from the right.
|
||||
let volume_rect = Rect::from_center_size(pos2(rect.right() - 14.0, rect.center().y), vec2(28.0, 28.0));
|
||||
volume(ui, id, volume_rect, audio);
|
||||
let time = format!("{} / {}", format_time(position), duration.map_or_else(|| "-:--".to_owned(), format_time));
|
||||
let galley = ui.painter().layout_no_wrap(time, FontId::new(11.0, theme::mono()), theme::MUTED);
|
||||
let time_x = volume_rect.left() - 8.0 - galley.size().x;
|
||||
ui.painter().galley(pos2(time_x, rect.center().y - galley.size().y / 2.0), galley, theme::MUTED);
|
||||
|
||||
// The waveform between them.
|
||||
let wave = Rect::from_min_max(pos2(button.right() + 10.0, rect.center().y - wave_h / 2.0), pos2(time_x - 10.0, rect.center().y + wave_h / 2.0));
|
||||
if wave.width() > 8.0 {
|
||||
let progress = duration.map_or(0.0, |d| (position / d).clamp(0.0, 1.0));
|
||||
let wave_resp = ui.interact(wave, id.with("wave"), Sense::click()).on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
match &waveform {
|
||||
WaveformLookup::Ready(w) => paint_bars(ui, wave, &audio::resample(&w.peaks, bar_count(wave.width())), progress),
|
||||
WaveformLookup::Loading => paint_bars(ui, wave, &placeholder_bars(url), progress),
|
||||
WaveformLookup::Failed => paint_line(ui, wave, progress),
|
||||
}
|
||||
if let Some(pointer) = wave_resp.interact_pointer_pos().filter(|_| wave_resp.clicked()) {
|
||||
let frac = ((pointer.x - wave.left()) / wave.width()).clamp(0.0, 1.0);
|
||||
audio.seek_track(&ctx, url, duration.map_or(0.0, |d| frac * d));
|
||||
}
|
||||
}
|
||||
rect
|
||||
}
|
||||
|
||||
/// "m:ss".
|
||||
pub fn format_time(secs: f32) -> String {
|
||||
let s = secs.max(0.0) as u64;
|
||||
format!("{}:{:02}", s / 60, s % 60)
|
||||
}
|
||||
|
||||
/// The volume button and its 0–100 slider flyout.
|
||||
fn volume(ui: &mut Ui, id: Id, rect: Rect, audio: &mut AudioPlayer) {
|
||||
let popup_id = id.with("volume-popup");
|
||||
let open = Popup::is_id_open(ui.ctx(), popup_id);
|
||||
let resp = ui.interact(rect, id.with("volume"), Sense::click()).on_hover_text("Volume").on_hover_cursor(egui::CursorIcon::PointingHand);
|
||||
resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, "Volume"));
|
||||
if resp.hovered() || open {
|
||||
ui.painter().rect_filled(rect, theme::RADIUS, theme::WASH);
|
||||
}
|
||||
let level = audio.track_volume();
|
||||
icons::paint(ui.painter(), rect.shrink(7.0), if level <= 0.0 { Icon::VolumeOff } else { Icon::Volume }, if resp.hovered() || open { theme::INK } else { theme::DIM });
|
||||
let mut percent = (level * 100.0).round() as u32;
|
||||
pickers::popover(&resp, popup_id, |ui| {
|
||||
ui.horizontal(|ui| {
|
||||
ui.add_space(6.0);
|
||||
ui.style_mut().spacing.slider_width = 120.0;
|
||||
ui.add(egui::Slider::new(&mut percent, 0..=100).show_value(false));
|
||||
ui.label(egui::RichText::new(percent.to_string()).size(11.0).family(theme::mono()).color(theme::MUTED));
|
||||
ui.add_space(4.0);
|
||||
});
|
||||
});
|
||||
if percent != (level * 100.0).round() as u32 {
|
||||
audio.set_track_volume(percent as f32 / 100.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn bar_count(width: f32) -> usize {
|
||||
((width + BAR_STEP - BAR_WIDTH) / BAR_STEP).floor().max(1.0) as usize
|
||||
}
|
||||
|
||||
/// Bars centred on the waveform's midline, blue up to `progress` (0–1).
|
||||
fn paint_bars(ui: &Ui, rect: Rect, heights: &[f32], progress: f32) {
|
||||
if heights.is_empty() {
|
||||
return;
|
||||
}
|
||||
let step = rect.width() / heights.len() as f32;
|
||||
let width = BAR_WIDTH.min(step * 0.7);
|
||||
let split = rect.left() + rect.width() * progress;
|
||||
let p = ui.painter();
|
||||
for (i, h) in heights.iter().enumerate() {
|
||||
let x = rect.left() + step * (i as f32 + 0.5);
|
||||
let half = (h.clamp(0.0, 1.0) * rect.height() / 2.0).max(1.0);
|
||||
let bar = Rect::from_center_size(pos2(x, rect.center().y), vec2(width, half * 2.0));
|
||||
p.rect_filled(bar, 1.0, if x <= split { PROGRESS_INK } else { WAVE_INK });
|
||||
}
|
||||
}
|
||||
|
||||
/// The fallback when the waveform can't be decoded: a thin track with the played part in blue.
|
||||
fn paint_line(ui: &Ui, rect: Rect, progress: f32) {
|
||||
let track = Rect::from_center_size(rect.center(), vec2(rect.width(), 4.0));
|
||||
ui.painter().rect_filled(track, 2.0, WAVE_INK);
|
||||
ui.painter().rect_filled(Rect::from_min_size(track.min, vec2(track.width() * progress, track.height())), 2.0, PROGRESS_INK);
|
||||
ui.painter().line_segment([pos2(track.left() + track.width() * progress, rect.top() + 6.0), pos2(track.left() + track.width() * progress, rect.bottom() - 6.0)], Stroke::new(1.0, PROGRESS_INK));
|
||||
}
|
||||
|
||||
/// Deterministic stand-in bars for a track (seeded by its URL), 25–89 % tall.
|
||||
pub fn placeholder_bars(url: &str) -> Vec<f32> {
|
||||
let mut seed = url.chars().fold(0_u32, |s, c| s.wrapping_mul(31).wrapping_add(c as u32));
|
||||
(0..PLACEHOLDER_BARS)
|
||||
.map(|_| {
|
||||
let percent = 25 + seed % 65;
|
||||
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
percent as f32 / 100.0
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A tile with a music note (`MusicIcon` in its box), for audio without artwork.
|
||||
pub fn music_tile(ui: &Ui, rect: Rect, round: bool) {
|
||||
let p = ui.painter();
|
||||
if round {
|
||||
p.circle_filled(rect.center(), rect.width() / 2.0, theme::WASH_HOVER);
|
||||
} else {
|
||||
p.rect(rect, theme::RADIUS, theme::WASH, theme::hairline(), egui::StrokeKind::Inside);
|
||||
}
|
||||
icons::paint(p, Rect::from_center_size(rect.center(), rect.size() * 0.42), Icon::Music, theme::fade(Color32::WHITE, 0.75));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn times_read_minutes_and_seconds() {
|
||||
assert_eq!(format_time(0.0), "0:00");
|
||||
assert_eq!(format_time(65.9), "1:05");
|
||||
assert_eq!(format_time(600.0), "10:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placeholder_bars_are_stable_per_track() {
|
||||
let a = placeholder_bars("https://cdn/a.mp3");
|
||||
assert_eq!(a.len(), PLACEHOLDER_BARS);
|
||||
assert_eq!(a, placeholder_bars("https://cdn/a.mp3"));
|
||||
assert_ne!(a, placeholder_bars("https://cdn/b.mp3"));
|
||||
assert!(a.iter().all(|h| (0.25..0.9).contains(h)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bars_fill_the_width() {
|
||||
assert_eq!(bar_count(2.0), 1);
|
||||
assert_eq!(bar_count(352.0), 101);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user