Compare commits

...
Author SHA1 Message Date
Brandon Thomas 6caced8820 manage api key in frontend 2025-05-13 08:15:19 -04:00
Brandon Thomas efef2ea206 end-to-end removal works! 2025-05-13 07:37:19 -04:00
Brandon Thomas 8d93104aa9 fix upload 2025-05-13 07:33:43 -04:00
Brandon Thomas f996ee85e9 almost end-to-end working 2025-05-13 07:24:26 -04:00
Brandon Thomas 3936ee4b6d endpoint handles media files and base64 images 2025-05-13 06:38:50 -04:00
Brandon Thomas 1998da9813 fal background removal endpoint 2025-05-13 06:10:03 -04:00
Brandon Thomas f01154f601 persist fal keys to disk 2025-05-13 05:33:18 -04:00
Brandon Thomas a9ccfdf9cd fal api key holder 2025-05-13 04:50:27 -04:00
Brandon Thomas c9ccb7680f remove background with fal 2025-05-13 04:31:35 -04:00
Brandon Thomas e0df1644a5 switch order of 2d/3d 2025-05-12 19:11:29 -04:00
Brandon Thomas 736d033ebf status alerts on 2d side 2025-05-12 18:54:42 -04:00
Brandon Thomas af3f0baa63 handle sora logged out state after app starts 2025-05-12 18:44:54 -04:00
Brandon Thomas a438dd13dc add Google Analytics GA4 2025-05-12 18:10:23 -04:00
Tanish Manku f7b76a7abe Add manual mouse support in Safari and AppleWebkit (#606) 2025-05-12 15:02:56 -07:00
Tanish Manku 9435af3e2a Change the relative pathing from base config (#605) 2025-05-12 14:23:09 -07:00
Tanish Manku 77efda1d66 Make 2D and 3D mount at the same time (#604) 2025-05-12 14:15:02 -07:00
Brandon Thomas fdc27b6ccd surface non-onboarding; do not retry non-onboarded requests 2025-05-12 17:00:25 -04:00
Brandon Thomas e3abd500db Explicit CSP on Mac instead of wildcards; this works on production Mac but may not fix Windows (#603) 2025-05-12 15:32:49 -04:00
Brandon Thomas ee4de4126d timeouts for uploads / remix (#602) 2025-05-12 12:26:41 -04:00
Brandon Thomas 4810ed30ae cache sora stats 2025-05-11 04:41:51 -04:00
Brandon Thomas 0b6593a063 fix preferences 2025-05-11 04:00:25 -04:00
Brandon Thomas b0e2dce140 fix event listeners for toasts; extra sounds (#601) 2025-05-11 03:41:14 -04:00
Brandon Thomas c4f401923b various sora / generate improvements (#600) 2025-05-11 01:02:00 -04:00
Brandon Thomas 24559a3554 username error 2025-05-09 19:00:28 -04:00
ArEnScandMichael Chung 5a26aed728 The bug was due to forggeting to bind this to the snapShotOfCurrentFr… (#599)
fix the generation problem

---------

Co-authored-by: Michael Chung <michaelchung@michaels-mbp-2.home>
2025-05-09 18:07:55 -04:00
Tanish Manku 95773c1c23 Point 3D tsconfig to the right subdirectory (#598) 2025-05-09 14:27:13 -07:00
109 changed files with 2380 additions and 6228 deletions
Generated
+157 -89
View File
@@ -463,7 +463,7 @@ dependencies = [
"getrandom 0.2.14",
"once_cell",
"version_check",
"zerocopy 0.7.33",
"zerocopy",
]
[[package]]
@@ -536,7 +536,7 @@ dependencies = [
"easyenv",
"enums",
"errors",
"futures 0.3.29",
"futures 0.3.31",
"hostname 0.4.0",
"log",
"mysql_queries",
@@ -646,9 +646,9 @@ checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e"
[[package]]
name = "anyhow"
version = "1.0.97"
version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
dependencies = [
"backtrace",
]
@@ -817,18 +817,20 @@ dependencies = [
"dirs 6.0.0",
"discord-rich-presence",
"errors",
"fal_client",
"filesys",
"futures 0.3.29",
"futures 0.3.31",
"image",
"indexmap 2.9.0",
"libloading 0.8.6",
"log",
"memory_caching",
"memory_store",
"mimetypes",
"once_cell",
"openai_sora_client",
"primitives",
"rand 0.9.0",
"rand 0.9.1",
"reqwest 0.12.15",
"serde",
"serde_derive",
@@ -1063,7 +1065,7 @@ dependencies = [
"serde_path_to_error",
"serde_qs 0.10.1",
"sha2 0.10.8",
"smart-default",
"smart-default 0.6.0",
"smol_str",
"thiserror 1.0.63",
"tokio",
@@ -1374,7 +1376,7 @@ dependencies = [
"serde_json",
"sqlx",
"tokio",
"toml 0.8.20",
"toml 0.8.22",
"url_config",
"user_traits_component",
"utoipa",
@@ -2052,7 +2054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02260d489095346e5cafd04dea8e8cb54d1d74fcd759022a9b72986ebe9a1257"
dependencies = [
"serde",
"toml 0.8.20",
"toml 0.8.22",
]
[[package]]
@@ -2155,9 +2157,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.40"
version = "0.4.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
dependencies = [
"android-tzdata",
"iana-time-zone",
@@ -3105,7 +3107,7 @@ dependencies = [
"easyenv",
"enums",
"errors",
"futures 0.3.29",
"futures 0.3.31",
"log",
"mysql_queries",
"serde",
@@ -3936,7 +3938,7 @@ dependencies = [
"cc",
"memchr",
"rustc_version 0.4.0",
"toml 0.8.20",
"toml 0.8.22",
"vswhom",
"winreg 0.52.0",
]
@@ -4024,7 +4026,7 @@ dependencies = [
"sqlx-core",
"sqlx-mysql",
"strum",
"toml 0.8.20",
"toml 0.8.22",
"utoipa",
"workspace-build-acceleration",
]
@@ -4038,7 +4040,7 @@ dependencies = [
"serde_derive",
"serde_json",
"strum",
"toml 0.8.20",
"toml 0.8.22",
"utoipa",
"workspace-build-acceleration",
]
@@ -4114,7 +4116,7 @@ dependencies = [
"elasticsearch_schema",
"enums",
"errors",
"futures 0.3.29",
"futures 0.3.31",
"hostname 0.4.0",
"log",
"mysql_queries",
@@ -4180,6 +4182,17 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "eventsource-stream"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab"
dependencies = [
"futures-core",
"nom",
"pin-project-lite",
]
[[package]]
name = "exr"
version = "1.73.0"
@@ -4201,6 +4214,51 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fal"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd7dd8565317310f05569296166e2cbdd2bc870173b2b24b77ed681166268d0"
dependencies = [
"base64 0.22.1",
"eventsource-stream",
"fal-derive",
"futures 0.3.31",
"image",
"reqwest 0.12.15",
"serde",
"serde_json",
"smart-default 0.7.1",
"thiserror 1.0.63",
]
[[package]]
name = "fal-derive"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f377f2f18a6bf107f907aa687df6bda2874bd38b25b1531f2a6fb38a9ca643"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "fal_client"
version = "0.0.1"
dependencies = [
"anyhow",
"errors",
"fal",
"file_to_base64_url",
"filesys",
"futures 0.3.31",
"testing",
"tokio",
"url",
"workspace-build-acceleration",
]
[[package]]
name = "fancy-regex"
version = "0.13.0"
@@ -4281,6 +4339,19 @@ dependencies = [
"rustc_version 0.4.0",
]
[[package]]
name = "file_to_base64_url"
version = "0.0.1"
dependencies = [
"anyhow",
"base64 0.22.1",
"errors",
"filesys",
"mimetypes",
"testing",
"workspace-build-acceleration",
]
[[package]]
name = "filesys"
version = "0.0.1"
@@ -4456,9 +4527,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
[[package]]
name = "futures"
version = "0.3.29"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
dependencies = [
"futures-channel",
"futures-core",
@@ -4471,9 +4542,9 @@ dependencies = [
[[package]]
name = "futures-channel"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
"futures-sink",
@@ -4481,15 +4552,15 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
[[package]]
name = "futures-executor"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
dependencies = [
"futures-core",
"futures-task",
@@ -4510,9 +4581,9 @@ dependencies = [
[[package]]
name = "futures-io"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-lite"
@@ -4544,9 +4615,9 @@ dependencies = [
[[package]]
name = "futures-macro"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
@@ -4555,21 +4626,21 @@ dependencies = [
[[package]]
name = "futures-sink"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
[[package]]
name = "futures-task"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
[[package]]
name = "futures-util"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-channel",
"futures-core",
@@ -5361,7 +5432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112fa2f6ad4ab815b9e1b938b4b1e437032d055e2f92ed10fd6ab2e62d02c6b6"
dependencies = [
"dirs 5.0.1",
"futures 0.3.29",
"futures 0.3.31",
"http 1.1.0",
"indicatif",
"log",
@@ -7147,7 +7218,7 @@ dependencies = [
"sqlx",
"tokens",
"tokio",
"toml 0.8.20",
"toml 0.8.22",
"workspace-build-acceleration",
]
@@ -7320,7 +7391,7 @@ dependencies = [
"directories 6.0.0",
"dirs 6.0.0",
"filesys",
"futures 0.3.29",
"futures 0.3.31",
"half",
"hf-hub 0.3.2",
"image",
@@ -7834,7 +7905,7 @@ dependencies = [
"log",
"serde",
"serde_derive",
"toml 0.8.20",
"toml 0.8.22",
]
[[package]]
@@ -8178,7 +8249,7 @@ dependencies = [
"sha2 0.10.8",
"tar",
"thiserror 1.0.63",
"toml 0.8.20",
"toml 0.8.22",
"ureq",
"url",
"uuid 1.8.0",
@@ -8240,7 +8311,7 @@ dependencies = [
"idempotency",
"log",
"once_cell",
"rand 0.9.0",
"rand 0.9.1",
"reqwest 0.12.15",
"serde",
"serde_derive",
@@ -8250,7 +8321,7 @@ dependencies = [
"testing",
"thiserror 2.0.12",
"tokio",
"toml 0.8.20",
"toml 0.8.22",
"workspace-build-acceleration",
]
@@ -9555,13 +9626,12 @@ dependencies = [
[[package]]
name = "rand"
version = "0.9.0"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.3",
"zerocopy 0.8.24",
]
[[package]]
@@ -9791,7 +9861,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6435842fc2fea44b528719eb8c32203bbc1bb2f5b619fbe0c0a3d8350fd8d2a8"
dependencies = [
"bytes 1.10.0",
"futures 0.3.29",
"futures 0.3.31",
"pin-project-lite",
]
@@ -10095,7 +10165,7 @@ dependencies = [
"serde",
"serde_derive",
"sqlx",
"toml 0.8.20",
"toml 0.8.22",
"workspace-build-acceleration",
]
@@ -10284,7 +10354,7 @@ dependencies = [
"base64 0.22.1",
"bytes 1.10.0",
"cfg-if 1.0.0",
"futures 0.3.29",
"futures 0.3.31",
"hex",
"hmac 0.12.1",
"http 1.1.0",
@@ -10977,7 +11047,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9"
dependencies = [
"futures 0.3.29",
"futures 0.3.31",
"log",
"once_cell",
"parking_lot 0.12.2",
@@ -11243,6 +11313,17 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "smart-default"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "smol_str"
version = "0.1.24"
@@ -11670,7 +11751,7 @@ dependencies = [
"errors",
"filesys",
"futures 0.1.31",
"futures 0.3.29",
"futures 0.3.31",
"futures-core",
"futures-util",
"google_sign_in",
@@ -11702,7 +11783,7 @@ dependencies = [
"prompt_engineering",
"pulldown-cmark 0.11.0",
"r2d2_redis",
"rand 0.9.0",
"rand 0.9.1",
"redis_caching",
"redis_common",
"redis_schema",
@@ -11728,7 +11809,7 @@ dependencies = [
"time",
"tokens",
"tokio",
"toml 0.8.20",
"toml 0.8.22",
"tts_common",
"url",
"url_config",
@@ -12393,7 +12474,7 @@ dependencies = [
"cfg-expr",
"heck 0.5.0",
"pkg-config",
"toml 0.8.20",
"toml 0.8.22",
"version-compare",
]
@@ -12538,7 +12619,7 @@ dependencies = [
"serde_json",
"tauri-utils",
"tauri-winres",
"toml 0.8.20",
"toml 0.8.22",
"walkdir",
]
@@ -12596,7 +12677,7 @@ dependencies = [
"serde",
"serde_json",
"tauri-utils",
"toml 0.8.20",
"toml 0.8.22",
"walkdir",
]
@@ -12636,7 +12717,7 @@ dependencies = [
"tauri-plugin",
"tauri-utils",
"thiserror 2.0.12",
"toml 0.8.20",
"toml 0.8.22",
"url",
"uuid 1.8.0",
]
@@ -12783,7 +12864,7 @@ dependencies = [
"serde_with 3.12.0",
"swift-rs",
"thiserror 2.0.12",
"toml 0.8.20",
"toml 0.8.22",
"url",
"urlpattern",
"uuid 1.8.0",
@@ -12797,7 +12878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56eaa45f707bedf34d19312c26d350bc0f3c59a47e58e8adbeecdc850d2c13a0"
dependencies = [
"embed-resource",
"toml 0.8.20",
"toml 0.8.22",
]
[[package]]
@@ -13123,16 +13204,16 @@ dependencies = [
"sqlx-core",
"sqlx-mysql",
"strum",
"toml 0.8.20",
"toml 0.8.22",
"utoipa",
"workspace-build-acceleration",
]
[[package]]
name = "tokio"
version = "1.43.0"
version = "1.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165"
dependencies = [
"backtrace",
"bytes 1.10.0",
@@ -13332,21 +13413,21 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.20"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit 0.22.24",
"toml_edit 0.22.26",
]
[[package]]
name = "toml_datetime"
version = "0.6.8"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3"
dependencies = [
"serde",
]
@@ -13388,17 +13469,24 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.22.24"
version = "0.22.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e"
dependencies = [
"indexmap 2.9.0",
"serde",
"serde_spanned",
"toml_datetime",
"toml_write",
"winnow 0.7.6",
]
[[package]]
name = "toml_write"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076"
[[package]]
name = "tonic"
version = "0.11.0"
@@ -15186,7 +15274,7 @@ dependencies = [
"digest 0.10.7",
"either",
"flate2",
"futures 0.3.29",
"futures 0.3.31",
"futures-channel",
"futures-core",
"futures-executor",
@@ -15504,16 +15592,7 @@ version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "087eca3c1eaf8c47b94d02790dd086cd594b912d2043d4de4bfdd466b3befb7c"
dependencies = [
"zerocopy-derive 0.7.33",
]
[[package]]
name = "zerocopy"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
dependencies = [
"zerocopy-derive 0.8.24",
"zerocopy-derive",
]
[[package]]
@@ -15527,17 +15606,6 @@ dependencies = [
"syn 2.0.87",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "zerofrom"
version = "0.1.5"
+9 -4
View File
@@ -30,6 +30,7 @@ members = [
"crates/lib/actix_cors_configs",
"crates/lib/actix_helpers",
"crates/lib/api_clients/cloud_storage",
"crates/lib/api_clients/fal_client",
"crates/lib/api_clients/google_drive_common",
"crates/lib/api_clients/google_sign_in",
"crates/lib/api_clients/openai_sora_client",
@@ -54,6 +55,7 @@ members = [
"crates/lib/email_sender",
"crates/lib/errors",
"crates/lib/files/archives",
"crates/lib/files/file_to_base64_url",
"crates/lib/files/filesys",
"crates/lib/files/hashing",
"crates/lib/files/images",
@@ -157,15 +159,16 @@ workspace-build-acceleration = { path = "crates/workspace-build-acceleration" }
# External libs
# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
anyhow = "1.0.97"
anyhow = "1.0.98"
base64 = "0.22.1"
chrono = { version = "0.4.40", features = ["serde"] }
chrono = { version = "0.4.41", features = ["serde"] }
cookie = "0.18.1"
futures = { version = "0.3.31" }
hex-literal = "1.0.0"
indexmap = "2.9.0"
log = "0.4.27"
once_cell = "1.21.3"
rand = "0.9.0"
rand = "0.9.1"
regex = "1.11.1"
reqwest = { version = "0.12.15", features = ["json", "cookies"] }
serde = { version = "1.0.219", features = ["derive"] }
@@ -175,7 +178,8 @@ serde_yaml = "0.9.1"
sha3 = "0.10.8"
tauri = { version = "2.4.1", features = ["unstable", "devtools"] } # NB: "devtools" prevents App Store distribution
thiserror = "2.0.12"
toml = "0.8.20"
tokio = { version = "1.45.0", features = ["macros"] }
toml = "0.8.22"
url = { version = "2.5.4", features = ["serde"] }
# Internal libs
@@ -183,6 +187,7 @@ crockford = { path = "crates/lib/data/crockford" }
enums = { path = "crates/_types/enums" }
enums_public = { path = "crates/_types/enums_public" }
errors = { path = "crates/lib/errors" }
fal_client = { path = "crates/lib/api_clients/fal_client" }
filesys = { path = "crates/lib/files/filesys" }
idempotency = { path = "crates/lib/idempotency" }
images = { path = "crates/lib/files/images" }
+2
View File
@@ -25,9 +25,11 @@ workspace-build-acceleration = { workspace = true }
# Internal
crockford = { workspace = true }
errors = { workspace = true }
fal_client = { workspace = true }
filesys = { workspace = true }
memory_caching = { workspace = true }
memory_store = { workspace = true }
mimetypes = { workspace = true }
openai_sora_client = { workspace = true }
primitives = { workspace = true }
storyteller_client = { workspace = true }
@@ -29,9 +29,10 @@ pub enum ValueType {
pub enum PreferenceName {
PreferredDownloadDirectory,
PlaySounds,
EnqueueSuccessSound,
EnqueueFailureSound,
GenerationSuccessSound,
GenerationFailureSound,
GenerationEnqueueSound,
}
#[derive(Serialize)]
@@ -85,6 +86,16 @@ async fn update_prefs(
return Err(anyhow!("Invalid value: {:?}", request.value)),
}
}
PreferenceName::EnqueueSuccessSound => {
prefs.enqueue_success_sound = request.value
.map(|val| string_value(&val))
.transpose()?;
}
PreferenceName::EnqueueFailureSound => {
prefs.enqueue_failure_sound = request.value
.map(|val| string_value(&val))
.transpose()?;
}
PreferenceName::GenerationSuccessSound => {
prefs.generation_success_sound = request.value
.map(|val| string_value(&val))
@@ -95,11 +106,6 @@ async fn update_prefs(
.map(|val| string_value(&val))
.transpose()?;
}
PreferenceName::GenerationEnqueueSound => {
prefs.generation_enqueue_sound = request.value
.map(|val| string_value(&val))
.transpose()?;
}
}
app_prefs.set_clone(&prefs)?;
@@ -3,7 +3,11 @@ use std::error::Error;
use std::str::FromStr;
/// Easy to use Result<T, E> type.
pub type CommandResult<T, E> = Result<CommandSuccessResponseWrapper<T>, CommandErrorResponseWrapper<E>>;
pub type CommandResult<SuccessPayload, ErrType, ErrPayload> =
Result<CommandSuccessResponseWrapper<SuccessPayload>, CommandErrorResponseWrapper<ErrType, ErrPayload>>;
/// No inner payloads for this type. Just strings as messages.
pub type SimpleCommandResult = CommandResult<(), (), ()>;
/// Statuses for successful commands.
#[derive(Serialize, Clone, Debug)]
@@ -18,6 +22,7 @@ pub enum CommandSuccessStatus {
pub enum CommandErrorStatus {
BadRequest, // 400
Unauthorized, // 401
TooManyRequests, // 429
ServerError, // 500
}
@@ -34,15 +39,22 @@ pub struct CommandSuccessResponseWrapper<T: Serialize> {
}
#[derive(Serialize, Debug)]
pub struct CommandErrorResponseWrapper<T: Serialize> {
pub struct CommandErrorResponseWrapper<ErrType: Serialize, ErrPayload: Serialize> {
/// Both "success" and "error" types have a `status` field for the frontend.
pub status: CommandErrorStatus,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_message: Option<String>,
/// Represents a "type" of error, preferably an enum without payloads.
/// Preferably this serializes to a string.
#[serde(skip_serializing_if = "Option::is_none")]
pub error_details: Option<T>,
pub error_type: Option<ErrType>,
/// Represents a full error details object or payload.
/// This could be a structure, message, or whatever.
#[serde(skip_serializing_if = "Option::is_none")]
pub error_details: Option<ErrPayload>,
}
@@ -61,7 +73,7 @@ impl CommandSuccessResponseWrapper<()> {
/// `Serialize`. Rust's `"specialization"`, which would let us account for this, is an unstable, nightly-only
/// feature. So instead we manually mark all types other than `()` with `SerializeMarker` as a hack to
/// distinguish the two.
trait SerializeMarker : Serialize {}
pub trait SerializeMarker : Serialize {}
// NB: () is Serialize! So we can't broadly implement just `T: Serialize`.
impl From<()> for CommandSuccessResponseWrapper<()> {
@@ -96,11 +108,12 @@ impl From<&str> for CommandSuccessResponseWrapper<String> {
}
}
impl From<&str> for CommandErrorResponseWrapper<String> {
impl From<&str> for CommandErrorResponseWrapper<(), ()> {
fn from(value: &str) -> Self {
CommandErrorResponseWrapper {
status: CommandErrorStatus::BadRequest, // NB: Default to 500 error.
error_message: Some(value.to_string()),
error_type: None,
error_details: None,
}
}
@@ -0,0 +1,296 @@
use crate::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker};
use crate::events::sendable_event_trait::SendableEvent;
use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::data_dir::trait_data_subdir::DataSubdir;
use crate::state::fal::fal_credential_manager::FalCredentialManager;
use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::state::sora::sora_credential_holder::SoraCredentialHolder;
use crate::state::sora::sora_credential_manager::SoraCredentialManager;
use crate::state::sora::sora_task_queue::SoraTaskQueue;
use crate::state::storyteller::storyteller_credential_manager::StorytellerCredentialManager;
use crate::utils::get_url_file_extension::get_url_file_extension;
use crate::utils::simple_http_download::simple_http_download;
use crate::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
use log::{debug, error, info, warn};
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use openai_sora_client::credentials::SoraCredentials;
use openai_sora_client::creds::credential_migration::CredentialMigrationRef;
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest};
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file;
use openai_sora_client::sora_error::SoraError;
use reqwest::Url;
use serde_derive::{Deserialize, Serialize};
use std::fs::read_to_string;
use std::io::{Cursor, Read, Write};
use std::ops::Add;
use std::path::PathBuf;
use std::time::Duration;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter, Manager, State};
use tempfile::NamedTempFile;
use filesys::file_read_bytes::file_read_bytes;
use tokens::tokens::media_files::MediaFileToken;
#[derive(Deserialize)]
pub struct FalBackgroundRemovalRequest {
/// Image media file; the image to remove the background from.
pub image_media_token: Option<MediaFileToken>,
/// Base64-encoded image
pub base64_image: Option<String>,
}
#[derive(Serialize)]
pub struct FalBackgroundRemovalSuccessResponse {
/// Result media token
pub media_token: MediaFileToken,
/// Result URL
pub cdn_url: Url,
/// Base64 bytes
pub base64_bytes: String,
}
impl SerializeMarker for FalBackgroundRemovalSuccessResponse {}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum SoraImageRemixErrorType {
/// Generic server error
ServerError,
/// No Fal API key available
NeedsFalApiKey,
}
#[tauri::command]
pub async fn fal_background_removal_command(
request: FalBackgroundRemovalRequest,
app_data_root: State<'_, AppDataRoot>,
fal_creds_manager: State<'_, FalCredentialManager>,
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
) -> CommandResult<FalBackgroundRemovalSuccessResponse, SoraImageRemixErrorType, ()> {
info!("fal_background_removal_command called; image media token: {:?}", request.image_media_token);
let has_credentials = fal_creds_manager
.has_apparent_api_token()
.unwrap_or(true); // NB: Failures would be lock issues
if !has_credentials {
warn!("No API key found");
return Err(CommandErrorResponseWrapper {
status: CommandErrorStatus::Unauthorized,
error_message: Some("You need to set a FAL api key".to_string()),
error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey),
error_details: None,
});
}
let result = remove_background(
request,
&app_data_root,
&fal_creds_manager,
&storyteller_creds_manager
).await;
match result {
Err(err) => {
error!("error: {:?}", err);
//let event = SoraImageEnqueueFailureEvent {};
//let result = event.send(&app);
//if let Err(err) = result {
// error!("Failed to emit event: {:?}", err);
//}
let mut status = CommandErrorStatus::ServerError;
let mut error_type = SoraImageRemixErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
//match (err) {
// _ => {},
//}
Err(CommandErrorResponseWrapper {
status,
error_message: Some(error_message.to_string()),
error_type: Some(error_type),
error_details: None,
})
}
Ok(result) => {
//let event = SoraImageEnqueueSuccessEvent {};
//let result = event.send(&app);
//if let Err(err) = result {
// error!("Failed to emit event: {:?}", err);
//}
Ok(FalBackgroundRemovalSuccessResponse {
media_token: result.0.media_file.token,
cdn_url: result.0.media_file.media_links.cdn_url,
base64_bytes: result.1,
}.into())
}
}
}
#[derive(Debug)]
enum InnerError {
FalError(FalErrorPlus),
AnyhowError(AnyhowError),
StorytellerApiError(ApiError),
DecodeError(DecodeError),
//ImageError(image::ImageError),
IoError(std::io::Error),
}
impl From<AnyhowError> for InnerError {
fn from(value: AnyhowError) -> Self {
Self::AnyhowError(value)
}
}
impl From<FalErrorPlus> for InnerError {
fn from(value: FalErrorPlus) -> Self {
Self::FalError(value)
}
}
impl From<ApiError> for InnerError {
fn from(value: ApiError) -> Self {
Self::StorytellerApiError(value)
}
}
impl From<DecodeError> for InnerError {
fn from(value: DecodeError) -> Self {
Self::DecodeError(value)
}
}
//impl From<image::ImageError> for InnerError {
// fn from(value: image::ImageError) -> Self {
// Self::ImageError(value)
// }
//}
//
impl From<std::io::Error> for InnerError {
fn from(value: std::io::Error) -> Self {
Self::IoError(value)
}
}
pub async fn remove_background(
request: FalBackgroundRemovalRequest,
app_data_root: &AppDataRoot,
fal_creds_manager: &FalCredentialManager,
storyteller_creds_manager: &StorytellerCredentialManager,
) -> Result<(GetMediaFileSuccessResponse, String), InnerError> {
let api_key = fal_creds_manager.get_key_required()?;
let creds = storyteller_creds_manager.get_credentials_required()?;
let mut temp_download;
if let Some(media_token) = request.image_media_token {
temp_download = download_media_file(&app_data_root, &media_token).await?;
} else if let Some(base64_bytes) = request.base64_image {
temp_download = persist_base64(&app_data_root, base64_bytes).await?;
} else {
return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided")));
}
info!("Calling FAL image background removal...");
let filename = temp_download.path().to_path_buf();
let result = remove_background_rembg(filename, &api_key).await?;
let extension_with_dot = get_url_file_extension(&result.image_url)
.map(|ext| format!(".{}", ext))
.unwrap_or_else(|| ".png".to_string());
let mut result_file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?;
info!("Downloading file result file...");
simple_http_download_to_tempfile(&result.image_url, &mut result_file).await?;
let result_filename = result_file.path().to_path_buf();
// NB: We couldn't pass the existing file handle as I think the file handle pointer is already at the EOF
let bytes = file_read_bytes(&result_filename)?;
let base64_bytes = BASE64_STANDARD.encode(bytes.as_bytes());
info!("Uploading image media file...");
let upload_result = upload_image_media_file_from_file(
&ApiHost::Storyteller,
Some(&creds),
result_filename
).await?;
// TODO: Don't re-request to simply build MediaLinks (or CDN URL). Get those from the upload API in one turn.
info!("Re-requesting media file...");
let response = get_media_file(&ApiHost::Storyteller, &upload_result.media_file_token).await?;
info!("Uploaded media file: {:?}", response.media_file.token);
Ok((response, base64_bytes))
}
async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result<NamedTempFile, InnerError> {
let response = get_media_file(&ApiHost::Storyteller, token).await?;
let media_file_url = &response.media_file.media_links.cdn_url;
let extension_with_dot = get_url_file_extension(media_file_url)
.map(|ext| format!(".{}", ext))
.unwrap_or_else(|| ".png".to_string());
//let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot);
//let filename = app_data_root.downloads_dir().path().join(&filename);
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?;
simple_http_download_to_tempfile(&media_file_url, &mut file).await?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result<NamedTempFile, InnerError> {
let bytes = BASE64_STANDARD.decode(base64_image)?;
let extension = MimetypeInfo::get_for_bytes(&bytes)
.map(|info| info.file_extension())
.flatten()
.map(|ext| ext.extension_with_period().to_string())
.unwrap_or_else(|| ".png".to_string());
let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?;
file.write_all(bytes.as_ref())?;
Ok(file) // NB: Must return TempFile to not drop / delete it
}
@@ -0,0 +1,46 @@
use crate::commands::command_response_wrapper::{CommandResult, SerializeMarker};
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
use crate::state::app_preferences::preferred_download_directory::PreferredDownloadDirectory;
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::fal::fal_credential_manager::FalCredentialManager;
use anyhow::anyhow;
use errors::AnyhowResult;
use fal_client::creds::fal_api_key::FalApiKey;
use log::{error, info};
use openai_sora_client::sora_error::SoraError::AnyhowError;
use primitives::traits::trim_or_emptyable::TrimOrEmptyable;
use serde_derive::{Deserialize, Serialize};
use tauri::{AppHandle, State};
#[derive(Serialize)]
pub struct GetFalApiKeyResponse {
pub key: Option<String>,
}
impl SerializeMarker for GetFalApiKeyResponse {}
#[tauri::command]
pub async fn get_fal_api_key_command(
creds_manager: State<'_, FalCredentialManager>,
) -> CommandResult<GetFalApiKeyResponse, (), ()> {
let key = get_key(&creds_manager)
.await
.map_err(|err| {
error!("Error getting API key: {:?}", err);
"error getting key"
})?;
Ok(key.into())
}
async fn get_key(
creds_manager: &FalCredentialManager,
) -> AnyhowResult<GetFalApiKeyResponse> {
let maybe_key = creds_manager.get_key()?
.map(|key| key.0)
.map(|key| key.trim().to_string())
.filter(|key| !key.is_empty());
Ok(GetFalApiKeyResponse { key: maybe_key })
}
@@ -0,0 +1,3 @@
pub mod fal_background_removal_command;
pub mod get_fal_api_key_command;
pub mod set_fal_api_key_command;
@@ -0,0 +1,59 @@
use crate::commands::command_response_wrapper::CommandResult;
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
use crate::state::app_preferences::preferred_download_directory::PreferredDownloadDirectory;
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::fal::fal_credential_manager::FalCredentialManager;
use anyhow::anyhow;
use errors::AnyhowResult;
use fal_client::creds::fal_api_key::FalApiKey;
use log::{error, info};
use openai_sora_client::sora_error::SoraError::AnyhowError;
use serde_derive::{Deserialize, Serialize};
use tauri::{AppHandle, State};
/// For now, we'll only update a single preference at a time.
#[derive(Deserialize)]
pub struct SetFalApiKeyRequest{
pub key: Option<String>,
}
#[tauri::command]
pub async fn set_fal_api_key_command(
request: SetFalApiKeyRequest,
creds_manager: State<'_, FalCredentialManager>,
) -> CommandResult<(), (), ()> {
info!("update_app_preferences_command called");
set_key(request, &creds_manager)
.await
.map_err(|err| {
error!("Error getting app preferences: {:?}", err);
"error setting key"
})?;
Ok(().into())
}
async fn set_key(
request: SetFalApiKeyRequest,
creds_manager: &FalCredentialManager,
) -> AnyhowResult<()> {
let key = request.key
.map(|key| key.trim().to_string())
.filter(|key| !key.is_empty())
.map(|key| FalApiKey::from_str(&key));
match key {
None => {
creds_manager.clear_key()?;
creds_manager.purge_api_key_from_disk()?;
}
Some(key) => {
creds_manager.set_key(&key)?;
creds_manager.persist_to_disk()?;
}
}
Ok(())
}
@@ -1,5 +1,6 @@
pub mod app_preferences;
pub mod command_response_wrapper;
pub mod fal;
pub mod flip_image;
pub mod platform_info_command;
pub mod sora;
@@ -1,3 +1,5 @@
use std::ops::Add;
use chrono::{DateTime, TimeDelta, Utc};
use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload;
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
use crate::state::data_dir::app_data_root::AppDataRoot;
@@ -12,6 +14,8 @@ use openai_sora_client::requests::list_media::list_media::list_media;
use serde_derive::Serialize;
use tauri::{AppHandle, State};
const CACHE_PERIOD : TimeDelta = TimeDelta::milliseconds(1000 * 60 * 5); // 5 minutes
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub struct CheckSoraSessionCommand {
@@ -63,10 +67,35 @@ async fn do_check(
});
}
};
let mut needs_upgrade_or_check = false;
let upgraded = maybe_upgrade_or_renew_session(&mut creds).await?;
let stats = sora_creds_manager.get_credential_stats()?;
match stats.last_consecutive_credential_success {
None => {
info!("Session never checked, needs upgrade or status checking.");
needs_upgrade_or_check= true;
}
Some(last_success) => {
let now = chrono::Utc::now();
let expires = last_success.add(CACHE_PERIOD);
if now.gt(&expires) {
info!("Session success expired, needs upgrade or status checking.");
needs_upgrade_or_check = true;
}
}
}
list_media(&creds).await?;
let mut upgraded = false;
if needs_upgrade_or_check {
info!("Attempting to upgrade session...");
upgraded = maybe_upgrade_or_renew_session(&mut creds).await?;
info!("Attempting to check session...");
list_media(&creds).await?;
sora_creds_manager.record_credential_success()?;
}
if upgraded {
info!("Saving upgraded session to manager and disk...");
@@ -14,8 +14,13 @@ use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_g
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use std::fs::read_to_string;
use std::io::Cursor;
use std::time::Duration;
use tauri::{AppHandle, Manager, State};
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
#[tauri::command]
pub async fn sora_image_generation_command(
prompt: &str,
@@ -55,8 +60,13 @@ pub async fn generate_image(
let image_bytes = BASE64_STANDARD.decode(image)?;
let filename = "image.png".to_string();
let response = sora_media_upload_from_bytes(image_bytes, filename, CredentialMigrationRef::New(&creds))
let response = sora_media_upload_from_bytes(
image_bytes,
filename,
CredentialMigrationRef::New(&creds),
Some(SORA_IMAGE_UPLOAD_TIMEOUT))
.await
.map_err(|err| {
error!("Failed to upload image to Sora: {:?}", err);
@@ -72,6 +82,7 @@ pub async fn generate_image(
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: CredentialMigrationRef::New(&creds),
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
}).await
.map_err(|err| {
error!("Failed to call Sora image generation: {:?}", err);
@@ -1,4 +1,4 @@
use crate::commands::command_response_wrapper::CommandResult;
use crate::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult};
use crate::events::sendable_event_trait::SendableEvent;
use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
@@ -12,10 +12,10 @@ use crate::utils::get_url_file_extension::get_url_file_extension;
use crate::utils::simple_http_download::simple_http_download;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use errors::AnyhowResult;
use errors::{AnyhowError, AnyhowResult};
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, ImageReader};
use log::{debug, error, info};
use log::{debug, error, info, warn};
use openai_sora_client::credentials::SoraCredentials;
use openai_sora_client::creds::credential_migration::CredentialMigrationRef;
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
@@ -25,14 +25,23 @@ use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file;
use openai_sora_client::sora_error::SoraError;
use serde_derive::{Deserialize, Serialize};
use std::fs::read_to_string;
use std::io::Cursor;
use std::ops::Add;
use std::time::Duration;
use storyteller_client::api_error::ApiError;
use storyteller_client::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::get_media_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter, Manager, State};
use tokens::tokens::media_files::MediaFileToken;
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
#[derive(Deserialize)]
pub struct SoraImageRemixCommand {
/// Image media file; the engine or canvas snapshot (screenshot).
@@ -50,6 +59,22 @@ pub struct SoraImageRemixCommand {
pub maybe_number_of_samples: Option<u32>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum SoraImageRemixErrorType {
/// Generic server error
ServerError,
/// The user is sending too many requests
TooManyConcurrentTasks,
/// User is not logged into Sora!
SoraLoginRequired,
/// The user needs to create a Sora account
SoraUsernameNotYetCreated,
/// The Sora service is having problems. Try again soon.
SoraIsHavingProblems,
}
#[tauri::command]
pub async fn sora_image_remix_command(
app: AppHandle,
@@ -57,36 +82,106 @@ pub async fn sora_image_remix_command(
app_data_root: State<'_, AppDataRoot>,
sora_creds_manager: State<'_, SoraCredentialManager>,
sora_task_queue: State<'_, SoraTaskQueue>,
) -> CommandResult<(), String> {
) -> CommandResult<(), SoraImageRemixErrorType, ()> {
info!("image_generation_command called; scene media token: {:?}, additional images: {:?}",
request.snapshot_media_token, request.maybe_additional_images);
// TODO(bt,2025-04-24): Better error messages to caller
let has_credentials = sora_creds_manager
.has_apparently_complete_credentials()
.unwrap_or(true); // NB: Failures would be lock issues
if !has_credentials {
warn!("No apparently completed credentials found");
return Err(CommandErrorResponseWrapper {
status: CommandErrorStatus::Unauthorized,
error_message: Some("You need to log into Sora to continue. See the settings menu.".to_string()),
error_type: Some(SoraImageRemixErrorType::SoraLoginRequired),
error_details: None,
});
}
let result = generate_image(request, &app_data_root, &sora_creds_manager, &sora_task_queue).await;
if let Err(err) = result {
error!("error: {:?}", err);
match result {
Err(err) => {
error!("error: {:?}", err);
let event = SoraImageEnqueueFailureEvent {};
let result = event.send(&app);
let event = SoraImageEnqueueFailureEvent {};
let result = event.send(&app);
if let Err(err) = result {
error!("Failed to emit event: {:?}", err);
if let Err(err) = result {
error!("Failed to emit event: {:?}", err);
}
let mut status = CommandErrorStatus::ServerError;
let mut error_type = SoraImageRemixErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
match (err) {
InnerError::SoraError(SoraError::TooManyConcurrentTasks) => {
error_type = SoraImageRemixErrorType::TooManyConcurrentTasks;
status = CommandErrorStatus::ServerError;
error_message = "You already have work in progress. Please wait for it to finish.";
},
InnerError::SoraError(SoraError::SoraUsernameNotYetCreated) => {
error_type = SoraImageRemixErrorType::SoraUsernameNotYetCreated;
status = CommandErrorStatus::BadRequest;
error_message = "You need to create a username on Sora.com to continue.";
},
InnerError::SoraError(SoraError::BadGateway(_) | SoraError::CloudFlareTimeout(_)) => {
error_type = SoraImageRemixErrorType::SoraIsHavingProblems;
status = CommandErrorStatus::ServerError;
error_message = "Sora is having problems. Please wait a moment and then retry.";
}
_ => {},
}
Err(CommandErrorResponseWrapper {
status,
error_message: Some(error_message.to_string()),
error_type: Some(error_type),
error_details: None,
})
}
Ok(_) => {
let event = SoraImageEnqueueSuccessEvent {};
let result = event.send(&app);
if let Err(err) = result {
error!("Failed to emit event: {:?}", err);
}
Ok(().into())
}
return Err("there was an error".into());
}
}
let event = SoraImageEnqueueSuccessEvent {};
let result = event.send(&app);
#[derive(Debug)]
enum InnerError {
SoraError(SoraError),
AnyhowError(AnyhowError),
StorytellerApiError(ApiError),
}
if let Err(err) = result {
error!("Failed to emit event: {:?}", err);
impl From<AnyhowError> for InnerError {
fn from(value: AnyhowError) -> Self {
Self::AnyhowError(value)
}
}
Ok(().into())
impl From<SoraError> for InnerError {
fn from(value: SoraError) -> Self {
Self::SoraError(value)
}
}
impl From<ApiError> for InnerError {
fn from(value: ApiError) -> Self {
Self::StorytellerApiError(value)
}
}
pub async fn generate_image(
@@ -94,7 +189,7 @@ pub async fn generate_image(
app_data_root: &AppDataRoot,
sora_creds_manager: &SoraCredentialManager,
sora_task_queue: &SoraTaskQueue,
) -> AnyhowResult<()> {
) -> Result<(), InnerError> {
let response = get_media_file(&ApiHost::Storyteller, &request.snapshot_media_token).await?;
@@ -133,6 +228,7 @@ pub async fn generate_image(
image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest {
file_path,
credentials: &creds,
request_timeout: Some(SORA_IMAGE_UPLOAD_TIMEOUT),
}).await?;
if let Some(new_creds) = maybe_new_credentials {
@@ -157,6 +253,7 @@ pub async fn generate_image(
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: &creds,
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
}).await?;
if let Some(new_creds) = maybe_new_creds {
@@ -1,5 +1,5 @@
use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload;
use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper};
use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper, SimpleCommandResult};
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::sora::sora_credential_manager::SoraCredentialManager;
@@ -17,7 +17,7 @@ use tauri::{AppHandle, State};
#[tauri::command]
pub async fn sora_logout_command(
sora_creds_manager: State<'_, SoraCredentialManager>,
) -> CommandResult<(), String> {
) -> SimpleCommandResult {
info!("sora_logout_command called");
if let Err(err) = sora_creds_manager.purge_credentials_from_disk() {
@@ -30,6 +30,10 @@ pub async fn sora_logout_command(
return Err("error clearing credentials from memory".into());
}
if let Err(err) = sora_creds_manager.reset_credential_stats() {
error!("Error resetting credential stats: {:?}", err);
return Err("error resetting credential stats".into());
}
Ok(().into())
}
+14 -1
View File
@@ -9,6 +9,9 @@ use tauri::Manager;
use crate::commands::app_preferences::get_app_preferences_command::get_app_preferences_command;
use crate::commands::app_preferences::update_app_preference_command::update_app_preferences_command;
use crate::commands::fal::fal_background_removal_command::fal_background_removal_command;
use crate::commands::fal::get_fal_api_key_command::get_fal_api_key_command;
use crate::commands::fal::set_fal_api_key_command::set_fal_api_key_command;
use crate::commands::flip_image::flip_image;
use crate::commands::platform_info_command::platform_info_command;
use crate::commands::sora::check_sora_session_command::check_sora_session_command;
@@ -18,6 +21,7 @@ use crate::commands::sora::sora_image_remix_command::sora_image_remix_command;
use crate::commands::sora::sora_logout_command::sora_logout_command;
use crate::state::app_preferences::app_preferences_manager::load_app_preferences_or_default;
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::fal::fal_credential_manager::FalCredentialManager;
use crate::state::main_window_position::MainWindowPosition;
use crate::state::main_window_size::MainWindowSize;
use crate::state::sora::sora_credential_manager::SoraCredentialManager;
@@ -47,10 +51,14 @@ pub fn run() {
println!("Attempting to read existing artcraft credentials...");
let storyteller_creds_manager = StorytellerCredentialManager::initialize_from_disk_infallible(&app_data_root);
let storyteller_creds_manager_2 = storyteller_creds_manager.clone();
let storyteller_creds_manager_3 = storyteller_creds_manager.clone();
println!("Attempting to read existing credentials...");
println!("Attempting to read existing sora credentials...");
let sora_creds_manager = SoraCredentialManager::initialize_from_disk_infallible(&app_data_root);
let sora_creds_manager_2 = sora_creds_manager.clone();
println!("Attempting to read existing fal credentials...");
let fal_creds_manager = FalCredentialManager::initialize_from_disk_infallible(&app_data_root);
// Other state
let sora_task_queue = SoraTaskQueue::new();
@@ -134,13 +142,18 @@ pub fn run() {
.manage(app_data_root)
.manage(app_preferences)
.manage(sora_creds_manager)
.manage(fal_creds_manager)
.manage(sora_task_queue)
.manage(storyteller_creds_manager_3)
.invoke_handler(tauri::generate_handler![
check_sora_session_command,
fal_background_removal_command,
flip_image,
get_app_preferences_command,
get_fal_api_key_command,
open_sora_login_command,
platform_info_command,
set_fal_api_key_command,
sora_image_generation_command,
sora_image_remix_command,
sora_logout_command,
@@ -40,4 +40,8 @@ impl AppCredentialsDir {
pub fn get_storyteller_session_cookie_file_path(&self) -> PathBuf {
self.path.join("artcraft_session.txt")
}
pub fn get_fal_api_key_file_path(&self) -> PathBuf {
self.path.join("fal_api_key.txt")
}
}
@@ -35,4 +35,17 @@ impl TemporaryDir {
.tempfile_in(&self.path)?;
Ok(tempfile)
}
pub fn new_named_temp_file_with_extension(&self, extension: &str) -> anyhow::Result<NamedTempFile> {
let extension = if extension.starts_with(".") {
extension.to_string()
} else {
format!(".{}", extension)
};
let tempfile = Builder::new()
.prefix("temp_")
.suffix(&extension)
.tempfile_in(&self.path)?;
Ok(tempfile)
}
}
@@ -0,0 +1,98 @@
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::fal::read_fal_api_key_from_disk::read_fal_api_key_from_disk;
use anyhow::anyhow;
use errors::AnyhowResult;
use fal_client::creds::fal_api_key::FalApiKey;
use memory_store::clone_slot::CloneSlot;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::Path;
use std::{fs, io};
#[derive(Clone)]
pub struct FalCredentialManager {
key: CloneSlot<FalApiKey>,
app_data_root: AppDataRoot,
}
impl FalCredentialManager {
pub fn new(app_data_root: &AppDataRoot) -> Self {
Self {
key: CloneSlot::empty(),
app_data_root: app_data_root.clone(),
}
}
pub fn initialize_from_disk_infallible(app_data_root: &AppDataRoot) -> Self {
let mut holder = CloneSlot::empty();
match read_fal_api_key_from_disk(app_data_root) {
Err(err) => {
log::warn!("Failed to read FAL API key from disk: {:?}", err);
}
Ok(api_key) => {
holder.set_clone(&api_key).expect("Failed to set FAL API key");
}
}
Self {
key: holder,
app_data_root: app_data_root.clone(),
}
}
pub fn set_key(&self, key: &FalApiKey) -> AnyhowResult<()> {
self.key.set_clone(key)
}
pub fn get_key(&self) -> AnyhowResult<Option<FalApiKey>> {
self.key.get_clone()
}
pub fn get_key_required(&self) -> AnyhowResult<FalApiKey> {
self.key.get_clone_required()
}
pub fn clear_key(&self) -> AnyhowResult<()> {
self.key.clear()
}
/// Does not check the token for validity
pub fn has_apparent_api_token(&self) -> AnyhowResult<bool> {
self.key.is_some()
}
pub fn purge_api_key_from_disk(&self) -> Result<(), io::Error> {
let filename = self.app_data_root.credentials_dir().get_fal_api_key_file_path();
if filename.exists() && filename.is_file() {
fs::remove_file(filename)?;
}
Ok(())
}
pub fn persist_to_disk(&self) -> AnyhowResult<()> {
let key = match self.get_key() {
Ok(Some(key)) => key,
Ok(None) => {
self.purge_api_key_from_disk()?;
return Ok(());
},
Err(err) => {
return Err(anyhow!("Failed to get FAL key: {:?}", err));
}
};
let filename = self.app_data_root.credentials_dir().get_fal_api_key_file_path();
let mut file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(filename)?;
file.write_all(key.0.as_bytes())?;
file.flush()?;
Ok(())
}
}
@@ -0,0 +1,2 @@
pub mod fal_credential_manager;
pub mod read_fal_api_key_from_disk;
@@ -0,0 +1,19 @@
use crate::state::data_dir::app_data_root::AppDataRoot;
use anyhow::anyhow;
use errors::AnyhowResult;
use fal_client::creds::fal_api_key::FalApiKey;
use std::fs::read_to_string;
pub fn read_fal_api_key_from_disk(app_data_root: &AppDataRoot) -> AnyhowResult<FalApiKey> {
let api_key_file = app_data_root.credentials_dir().get_fal_api_key_file_path();
if !api_key_file.exists() {
return Err(anyhow!("API key file does not exist: {:?}", api_key_file));
}
let value = read_to_string(api_key_file)?
.trim()
.to_string();
Ok(FalApiKey::new(value))
}
+1
View File
@@ -2,6 +2,7 @@ pub mod app_preferences;
pub mod app_startup_time;
pub mod data_dir;
pub mod expanduser;
pub mod fal;
pub mod main_window_position;
pub mod main_window_size;
pub mod os_platform;
@@ -2,4 +2,5 @@ pub mod read_sora_credentials_from_disk;
pub mod read_sora_credentials_from_disk_with_upgrade;
pub mod sora_credential_holder;
pub mod sora_credential_manager;
pub mod sora_credential_stats;
pub mod sora_task_queue;
@@ -54,4 +54,20 @@ impl SoraCredentialHolder {
}
}
}
/// This does not guarantee the credentials are correct, just that they're set.
pub fn has_apparently_complete_credentials(&self) -> AnyhowResult<bool> {
match self.credentials.read() {
Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
Ok(creds) => match &*creds {
None => Ok(false),
Some(creds) => {
let has_cookies = !creds.cookies.as_str().is_empty();
let has_bearer = creds.jwt_bearer_token.is_some();
let has_sentinel = creds.sora_sentinel.is_some();
Ok(has_cookies && has_bearer && has_sentinel)
},
}
}
}
}
@@ -1,23 +1,29 @@
use std::{fs, io};
use crate::state::data_dir::app_data_root::AppDataRoot;
use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::state::sora::sora_credential_holder::SoraCredentialHolder;
use crate::state::sora::sora_credential_stats::SoraCredentialStats;
use anyhow::anyhow;
use errors::AnyhowResult;
use log::{error, info, warn};
use memory_store::clone_cell::CloneCell;
use openai_sora_client::creds::sora_cookies::SoraCookies;
use openai_sora_client::creds::sora_credential_set::SoraCredentialSet;
use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
use openai_sora_client::creds::sora_sentinel::SoraSentinel;
use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::Path;
use openai_sora_client::creds::sora_cookies::SoraCookies;
use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
use std::{fs, io};
#[derive(Clone)]
pub struct SoraCredentialManager {
holder: SoraCredentialHolder,
app_data_root: AppDataRoot,
/// This is meant to help debug credential issues, keep credentials cached, fresh, etc.
// TODO: Put this into the same structure as the credentials and also persist to disk.
stats: CloneCell<SoraCredentialStats>,
}
impl SoraCredentialManager {
@@ -35,9 +41,41 @@ impl SoraCredentialManager {
Self {
holder,
app_data_root: app_data_root.clone(),
stats: CloneCell::with_owned(SoraCredentialStats::new()),
}
}
pub fn record_credential_success(&self) -> AnyhowResult<()> {
let mut stats = self.stats.get_clone()?;
stats.record_credential_success();
self.stats.set_owned(stats)?;
Ok(())
}
pub fn record_credential_failure(&self) -> AnyhowResult<()> {
let mut stats = self.stats.get_clone()?;
stats.record_credential_failure();
self.stats.set_owned(stats)?;
Ok(())
}
pub fn reset_credential_stats(&self) -> AnyhowResult<()> {
self.stats.set_owned(SoraCredentialStats::new())
}
/// This does not guarantee the credentials are correct
pub fn has_apparently_complete_credentials(&self) -> AnyhowResult<bool> {
self.holder.has_apparently_complete_credentials()
}
pub fn get_credential_stats(&self) -> AnyhowResult<SoraCredentialStats> {
self.stats.get_clone()
}
pub fn set_credential_stats(&self, stats: SoraCredentialStats) -> AnyhowResult<()> {
self.stats.set_owned(stats)
}
pub fn set_credentials(&self, creds: &SoraCredentialSet) -> AnyhowResult<()> {
self.holder.set_credentials(creds)
}
@@ -0,0 +1,47 @@
/// This is meant to track 401s and other credential failures, not 500s.
#[derive(Clone)]
pub struct SoraCredentialStats {
pub last_credential_failure: Option<chrono::DateTime<chrono::Utc>>,
pub last_credential_success: Option<chrono::DateTime<chrono::Utc>>,
pub credential_success_count: u32,
pub credential_failure_count: u32,
/// This resets on each failure
pub last_consecutive_credential_success: Option<chrono::DateTime<chrono::Utc>>,
/// This resets on each failure
pub consecutive_credential_success_count: u32,
/// This resets on each success
pub consecutive_credential_failure_count: u32,
}
impl SoraCredentialStats {
pub fn new() -> Self {
Self {
last_credential_failure: None,
last_credential_success: None,
last_consecutive_credential_success: None,
credential_success_count: 0,
credential_failure_count: 0,
consecutive_credential_success_count: 0,
consecutive_credential_failure_count: 0,
}
}
pub fn record_credential_success(&mut self) {
let now = chrono::Utc::now();
self.last_credential_success = Some(now);
self.last_consecutive_credential_success = Some(now);
self.credential_success_count += 1;
self.consecutive_credential_success_count += 1;
self.consecutive_credential_failure_count = 0;
}
pub fn record_credential_failure(&mut self) {
let now = chrono::Utc::now();
self.last_credential_failure = Some(now);
self.last_consecutive_credential_success = None;
self.credential_failure_count += 1;
self.consecutive_credential_failure_count += 1;
self.consecutive_credential_success_count = 0;
}
}
+1
View File
@@ -5,5 +5,6 @@ pub mod image;
pub mod large_file_transfer;
pub mod object_store;
pub mod simple_http_download;
pub mod simple_http_download_to_tempfile;
pub mod sora;
pub mod webview_unsafe;
@@ -0,0 +1,26 @@
use errors::AnyhowResult;
use reqwest::Client;
use reqwest::Url;
use std::io::Write;
use tempfile::NamedTempFile;
const USER_AGENT: &str = "storyteller-client/1.0";
/// Downloads a (binary) file to a filesystem path. Good for images, etc. Not great for large files.
pub async fn simple_http_download_to_tempfile(url: &Url, temp_file: &mut NamedTempFile) -> AnyhowResult<()> {
let client = Client::builder()
.gzip(true)
.build()?;
let response = client.get(url.clone()) // NB: No IntoUrl for &Url.
.header("User-Agent", USER_AGENT)
.header("Accept", "*/*")
.send()
.await?;
let bytes = response.bytes().await?;
temp_file.write_all(&bytes)?;
Ok(())
}
+37 -3
View File
@@ -21,11 +21,45 @@
],
"security": {
"csp": {
"default-src": "*",
"connect-src": "*",
"default-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"blob:",
"data:",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://www.googletagmanager.com",
"ipc://localhost",
"tauri://localhost"
],
"connect-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"blob:",
"data:",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://unpkg.com/detect-gpu@5.0.70/",
"https://www.googletagmanager.com",
"ipc://localhost",
"tauri://localhost"
],
"font-src": "*",
"img-src": "* data: blob: 'unsafe-inline'",
"style-src": "*"
"style-src": [
"'self'",
"'unsafe-eval'",
"'unsafe-inline'",
"blob:",
"data:",
"https://*.fakeyou.com",
"https://*.storyteller.ai",
"https://fonts.googleapis.com",
"ipc://localhost",
"tauri://localhost"
]
}
}
},
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
{"default":{"identifier":"default","description":"enables the default permissions","local":true,"windows":["main"],"permissions":["core:default","log:default","upload:allow-download","upload:default",{"identifier":"http:default","allow":[{"url":"https://*.storyteller.ai"},{"url":"https://*.fakeyou.com"}],"deny":[{"url":"https://private.tauri.app"}]}]}}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,32 @@
[package]
name = "fal_client"
edition = "2021"
version = "0.0.1"
publish = false
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
[lib]
name = "fal_client"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration = { workspace = true }
# Internal
file_to_base64_url = { path = "../../files/file_to_base64_url" }
errors = { workspace = true }
# External
anyhow = { workspace = true }
fal = { version = "0.3.6", features = ["endpoints"] }
futures = { workspace = true }
tokio = { workspace = true }
url = { workspace = true }
[dev-dependencies]
testing = { path = "../../testing" }
filesys = { path = "../../files/filesys" }
@@ -0,0 +1,13 @@
#[derive(Clone)]
pub struct FalApiKey(pub String);
impl FalApiKey {
pub fn new(api_key: String) -> Self {
Self(api_key)
}
pub fn from_str(api_key: &str) -> Self {
Self(api_key.to_string())
}
}
@@ -0,0 +1 @@
pub mod fal_api_key;
@@ -0,0 +1,43 @@
use std::error::Error;
use std::fmt::{Display, Formatter};
/// Additional errors that aren't included in `fal::FalError`.
#[derive(Debug)]
pub enum FalErrorPlus {
/// An error arising in the `fal` crate.
FalError(fal::FalError),
/// Another error we didn't handle.
AnyhowError(anyhow::Error),
/// URL parse errors.
UrlParseError(url::ParseError),
}
impl Display for FalErrorPlus {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::FalError(err) => write!(f, "FalErrorPlus::FalError: {:?}", err),
Self::AnyhowError(err) => write!(f, "FalErrorPlus::AnyhowError: {:?}", err),
Self::UrlParseError(err) => write!(f, "FalErrorPlus::UrlParseError: {:?}", err),
}
}
}
impl Error for FalErrorPlus {}
impl From<fal::FalError> for FalErrorPlus {
fn from(err: fal::FalError) -> Self {
FalErrorPlus::FalError(err)
}
}
impl From<anyhow::Error> for FalErrorPlus {
fn from(err: anyhow::Error) -> Self {
FalErrorPlus::AnyhowError(err)
}
}
impl From<url::ParseError> for FalErrorPlus {
fn from(err: url::ParseError) -> Self {
FalErrorPlus::UrlParseError(err)
}
}
@@ -0,0 +1,7 @@
//! A wrapper for the "fal" crate that bakes in a few extra recipes and utilities.
extern crate fal;
pub mod creds;
pub mod fal_error_plus;
pub mod requests;
@@ -0,0 +1 @@
pub mod remove_background_rembg;
@@ -0,0 +1,79 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use fal::endpoints::fal_ai::imageutils::rembg::RemoveBackgroundInput;
use fal::prelude::fal_ai::imageutils::rembg::rembg;
use fal::prelude::*;
use fal_client::file_to_base64_url::file_to_base64_url;
use futures::StreamExt;
use std::path::Path;
use url::Url;
pub struct RemBgArgs<'a, P: AsRef<Path>> {
pub image_path: P,
pub api_key: &'a FalApiKey,
}
pub struct RemBgResponse {
pub image_url: Url,
}
/// remove background using `fal_ai/imageutils/rembg`
pub async fn remove_background_rembg<P: AsRef<Path>>(image_path: P, api_key: &FalApiKey) -> Result<RemBgResponse, FalErrorPlus> {
let image_url = file_to_base64_url(image_path)?;
/*
TODO: Handle error messages -
FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected '<key_id>:<key_secret>'.\"}")))
FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}")))
*/
let request = RemoveBackgroundInput {
image_url,
crop_to_bbox: None,
sync_mode: None
};
let result = rembg(request)
.with_api_key(&api_key.0)
.queue()
.await?;
let mut stream = result.stream(true).await?;
while let Some(status) = stream.next().await {
let status = status?;
if status.status == Status::Completed {
break;
}
}
let output = result.response().await?;
let url = Url::parse(&output.image.url)?;
Ok(RemBgResponse {
image_url: url,
})
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::remove_background_rembg::remove_background_rembg;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
#[tokio::test]
#[ignore]
async fn test_remove_background_rembg() -> AnyhowResult<()> {
let image = test_file_path("test_data/image/juno.jpg")?;
// TODO: DO NOT COMMIT SECRET !
const KEY : &str = "DO NOT COMMIT";
let api_key = FalApiKey::from_str(KEY);
let result = remove_background_rembg(image, &api_key).await?;
Ok(())
}
}
@@ -1,3 +1,4 @@
use std::time::Duration;
use anyhow::anyhow;
use log::{info, warn};
use errors::AnyhowResult;
@@ -7,7 +8,7 @@ use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
use crate::creds::sora_sentinel::SoraSentinel;
use crate::requests::bearer::generate::generate_bearer_with_cookie;
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::requests::image_gen::image_gen_http_request;
use crate::requests::image_gen::{image_gen_http_request, SoraImageGenError};
use crate::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use crate::requests::sentinel_refresh::generate::token::generate_token;
use crate::sora_error::SoraError;
@@ -18,6 +19,8 @@ pub struct ImageRemixAutoRenewRequest<'a> {
pub image_size: ImageSize,
pub sora_media_tokens: Vec<String>,
pub credentials: &'a SoraCredentialSet,
/// This function can try several different requests. This is the timeout for *individual* requests.
pub request_timeout: Option<Duration>,
}
/// Image request with retry and session auto-renewal.
@@ -29,6 +32,7 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
image_size: request.image_size,
sora_media_tokens: request.sora_media_tokens.clone(), // NB: Clone because used again
credentials: CredentialMigrationRef::New(request.credentials),
request_timeout: request.request_timeout,
}).await;
let err = match result {
@@ -43,29 +47,32 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
match err {
// We'll fail these requests...
image_gen_http_request::SoraError::TooManyConcurrentTasks(_) => {
SoraImageGenError::TooManyConcurrentTasks(_) => {
return Err(SoraError::TooManyConcurrentTasks);
}
image_gen_http_request::SoraError::GenericError(err) => {
SoraImageGenError::GenericError(err) => {
return Err(SoraError::OtherBadStatus(anyhow!("image remix failed with GenericError: {:?}", err)))
}
image_gen_http_request::SoraError::NetworkError(err) => {
SoraImageGenError::NetworkError(err) => {
// TODO: The underlying type should be a reqwest::Error.
return Err(SoraError::OtherBadStatus(anyhow!("network error: {:?}", err)))
}
SoraImageGenError::UsernameRequired(_err) => {
return Err(SoraError::SoraUsernameNotYetCreated)
}
// We'll retry these requests...
image_gen_http_request::SoraError::SentinelBlock(err) => {
SoraImageGenError::SentinelBlock(err) => {
warn!("Image generation failed due to sentinel block error: {:?}", err);
refresh_sentinel = true;
}
image_gen_http_request::SoraError::TokenExpired(err) => {
SoraImageGenError::TokenExpired(err) => {
warn!("Image generation failed due to token expired error: {:?}", err);
refresh_sentinel = true; // TODO: Not sure what this error is, actually.
refresh_jwt = true;
}
image_gen_http_request::SoraError::InvalidJwt(err) => {
SoraImageGenError::InvalidJwt(err) => {
warn!("Image generation failed due to invalid jwt error: {:?}", err);
refresh_jwt = true;
}
@@ -123,12 +130,13 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
image_size: request.image_size,
sora_media_tokens: request.sora_media_tokens,
credentials: CredentialMigrationRef::New(&new_creds),
request_timeout: request.request_timeout,
}).await;
match result {
Ok(response) => Ok((response, Some(new_creds))),
Err(err) => match err {
image_gen_http_request::SoraError::TooManyConcurrentTasks(err) => {
image_gen_http_request::SoraImageGenError::TooManyConcurrentTasks(err) => {
Err(SoraError::TooManyConcurrentTasks)
}
_ => {
@@ -14,12 +14,16 @@ use crate::sora_error::SoraError;
use anyhow::anyhow;
use errors::AnyhowResult;
use log::{error, info, warn};
use std::time::Duration;
pub struct ImageUploadFromBytesAutoRenewRequest<'a> {
pub file_bytes: Vec<u8>,
pub filename: String,
pub mime_type: &'a str,
pub credentials: &'a SoraCredentialSet,
/// This function can try to upload several times. This is the timeout for *individual* requests.
pub request_timeout: Option<Duration>,
}
/// Image upload with retry and session auto-renewal.
@@ -32,6 +36,7 @@ pub async fn image_upload_from_bytes_with_session_auto_renew(
request.file_bytes.clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :(
request.filename.clone(), // FIXME: Same
CredentialMigrationRef::New(request.credentials),
request.request_timeout,
).await;
let err = match result {
@@ -51,6 +56,7 @@ pub async fn image_upload_from_bytes_with_session_auto_renew(
request.file_bytes,
request.filename,
CredentialMigrationRef::New(&new_creds),
request.request_timeout,
).await?;
Ok((result, Some(new_creds)))
@@ -6,10 +6,14 @@ use crate::requests::upload::upload_media_http_request::{upload_media_http_reque
use crate::sora_error::SoraError;
use log::{info, warn};
use std::path::Path;
use std::time::Duration;
pub struct ImageUploadFromFileAutoRenewRequest<'a, P: AsRef<Path>> {
pub file_path: P,
pub credentials: &'a SoraCredentialSet,
/// This function can try to upload several times. This is the timeout for *individual* requests.
pub request_timeout: Option<Duration>,
}
/// Image upload with retry and session auto-renewal.
@@ -21,10 +25,15 @@ pub async fn image_upload_from_file_with_session_auto_renew<P: AsRef<Path>>(
let result = sora_media_upload_from_file(
request.file_path.as_ref().clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :(
CredentialMigrationRef::New(request.credentials),
/// This function can try to upload several times. This is the timeout for *individual* requests.
request.request_timeout,
).await;
let err = match result {
Ok(response) => return Ok((response, None)),
// We can't retry some errors.
Err(err @ SoraError::SoraUsernameNotYetCreated) => return Err(err),
// Retry all other errors.
Err(err) => err,
};
@@ -37,6 +46,7 @@ pub async fn image_upload_from_file_with_session_auto_renew<P: AsRef<Path>>(
let result = sora_media_upload_from_file(
request.file_path,
CredentialMigrationRef::New(&new_creds),
request.request_timeout,
).await?;
Ok((result, Some(new_creds)))
@@ -39,11 +39,10 @@ pub async fn list_sora_task_status_with_session_auto_renew(
new_creds = Some(updated_creds);
}
let legacy_creds = new_creds.as_ref()
.unwrap_or_else(|| &args.credentials)
.to_legacy_credentials()?;
let upgraded_creds = new_creds.as_ref()
.unwrap_or_else(|| &args.credentials);
let result = get_image_gen_status(&request, &legacy_creds).await;
let result = get_image_gen_status(&request, &upgraded_creds).await;
let err = match result {
Ok(response) => return Ok((response, new_creds)),
@@ -56,7 +55,7 @@ pub async fn list_sora_task_status_with_session_auto_renew(
info!("Retrying image upload with new credentials...");
let result = get_image_gen_status(&request, &legacy_creds).await?;
let result = get_image_gen_status(&request, &upgraded_creds).await?;
Ok((result, Some(new_creds)))
}
@@ -2,6 +2,7 @@ use crate::credentials::SoraCredentials;
use crate::requests::image_gen::image_gen_status::TaskStatus;
use crate::requests::image_gen::image_gen_status::{get_image_gen_status, StatusRequest, TaskResponse, VideoGenStatusResponse};
use errors::AnyhowResult;
use crate::creds::sora_credential_set::SoraCredentialSet;
pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCredentials, retry_limit: Option<u32>) -> AnyhowResult<TaskResponse> {
let status_request = StatusRequest {
@@ -10,11 +11,13 @@ pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCrede
// before: Some(task_id.clone()),
};
let retry_limit = retry_limit.unwrap_or(10);
let new_creds = SoraCredentialSet::from_legacy_credentials(&credentials)?;
for _ in 0..retry_limit {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
let status_response = get_image_gen_status(&status_request, credentials).await?;
let status_response = get_image_gen_status(&status_request, &new_creds).await?;
match find_task_response_by_id(&status_response, task_id.clone()) {
Some(task_response) => match TaskStatus::from_str(&task_response.status) {
@@ -1,7 +1,9 @@
use crate::creds::credential_migration::CredentialMigrationRef;
use serde_derive::{Deserialize, Serialize};
use thiserror::Error;
use crate::requests::image_gen::image_gen_status::TaskId;
use log::warn;
use serde_derive::{Deserialize, Serialize};
use std::time::Duration;
use thiserror::Error;
const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
@@ -9,7 +11,7 @@ const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
#[derive(Error, Debug)]
pub enum SoraError {
pub enum SoraImageGenError {
#[error("Sora token expired: {0}")]
TokenExpired(String),
@@ -27,6 +29,9 @@ pub enum SoraError {
#[error("Network error: {0}")]
NetworkError(String),
#[error("Sora username required: {0}")]
UsernameRequired(String),
}
#[derive(Serialize, Debug)]
@@ -127,6 +132,7 @@ pub enum SoraErrorCode {
InvalidJwt,
TokenExpired,
TooManyConcurrentTasks,
UsernameRequired,
Unknown(String),
}
@@ -147,7 +153,11 @@ impl std::fmt::Display for RawSoraErrorInner {
}
/// Don't expose the internal request implementation as there are only a few "correct" ways to call the API.
pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: CredentialMigrationRef<'_>) -> Result<RawSoraResponse, SoraError> {
pub (crate) async fn image_gen_http_request(
sora_request: RawSoraImageGenRequest,
credentials: CredentialMigrationRef<'_>,
request_timeout: Option<Duration>,
) -> Result<RawSoraResponse, SoraImageGenError> {
let client = reqwest::Client::new();
let mut cookie;
@@ -161,78 +171,88 @@ pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest
// TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently.
sentinel = creds.sentinel.as_ref()
.map(|sentinel| sentinel.to_string())
.ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
.ok_or(SoraImageGenError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
}
CredentialMigrationRef::New(creds) => {
cookie = creds.cookies.to_string();
// TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently.
authorization_header = creds.jwt_bearer_token.as_ref()
.ok_or(SoraError::InvalidJwt("JWT bearer is required for image generation".to_string()))?
.ok_or(SoraImageGenError::InvalidJwt("JWT bearer is required for image generation".to_string()))?
.to_authorization_header_value();
// TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently.
sentinel = creds.sora_sentinel.as_ref()
.map(|sentinel| sentinel.get_sentinel().to_string())
.ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
.ok_or(SoraImageGenError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
}
}
let http_request = client.post(SORA_IMAGE_GEN_URL)
let mut http_request = client.post(SORA_IMAGE_GEN_URL)
.header("User-Agent", USER_AGENT)
.header("Cookie", &cookie)
.header("Authorization", &authorization_header)
.header("Content-Type", "application/json")
.header("OpenAI-Sentinel-Token", &sentinel);
if let Some(timeout) = request_timeout {
http_request = http_request.timeout(timeout);
}
let http_request = http_request.json(&sora_request).build()
.map_err(|e| SoraError::NetworkError(e.to_string()))?;
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
let response = client.execute(http_request)
.await
.map_err(|e| SoraError::NetworkError(e.to_string()))?;
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
let status = response.status();
let response_body = &response.text().await
.map_err(|e| SoraError::NetworkError(e.to_string()))?;
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
if status != reqwest::StatusCode::OK {
warn!("Sora image generation failure. Raw response: {:?}", response_body);
let error_response: RawSoraErrorResponse = serde_json::from_str(response_body)
.map_err(|e| SoraError::GenericError(format!("Failed to parse error response: {}", e)))?;
.map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse error response: {}", e)))?;
// Check for specific error codes
if let Some(code) = &error_response.error.code {
match code {
SoraErrorCode::TokenExpired => {
return Err(SoraError::TokenExpired(error_response.error.message));
return Err(SoraImageGenError::TokenExpired(error_response.error.message));
}
SoraErrorCode::SentinelBlock => {
return Err(SoraError::SentinelBlock(error_response.error.message));
return Err(SoraImageGenError::SentinelBlock(error_response.error.message));
}
SoraErrorCode::TooManyConcurrentTasks => {
return Err(SoraError::TooManyConcurrentTasks(error_response.error.message));
return Err(SoraImageGenError::TooManyConcurrentTasks(error_response.error.message));
}
SoraErrorCode::Unknown(_) => {
return Err(SoraError::GenericError(error_response.error.message));
return Err(SoraImageGenError::GenericError(error_response.error.message));
}
SoraErrorCode::InvalidJwt => {
return Err(SoraError::InvalidJwt(error_response.error.message));
return Err(SoraImageGenError::InvalidJwt(error_response.error.message));
}
SoraErrorCode::UsernameRequired => {
return Err(SoraImageGenError::UsernameRequired(error_response.error.message));
}
}
}
return Err(SoraError::GenericError(error_response.error.message));
return Err(SoraImageGenError::GenericError(error_response.error.message));
}
let response = serde_json::from_str(response_body)
.map_err(|e| SoraError::GenericError(format!("Failed to parse success response: {}", e)))?;
.map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse success response: {}", e)))?;
Ok(response)
}
#[cfg(test)]
mod tests {
use errors::AnyhowResult;
use crate::requests::image_gen::image_gen_http_request::RawSoraResponse;
use errors::AnyhowResult;
#[test]
fn deserialize_task_id() -> AnyhowResult<()> {
@@ -6,7 +6,9 @@ use reqwest::Url;
use serde_derive::{Deserialize, Serialize};
use std::fmt::{Debug, Display};
use std::{fs::File, io::Write, path::Path};
use log::debug;
use log::{debug, warn};
use crate::creds::sora_credential_set::SoraCredentialSet;
use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
const SORA_STATUS_URL: &str = "https://sora.com/backend/video_gen";
@@ -171,7 +173,7 @@ pub struct InpaintItem {
pub r#type: String,
pub preset_id: Option<String>,
pub generation_id: Option<String>,
pub upload_media_id: String,
pub upload_media_id: Option<String>,
pub frame_index: i32,
pub source_start_frame: i32,
}
@@ -199,7 +201,16 @@ impl StatusRequest {
}
/// Gets the status of image generation tasks from Sora API
pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentials) -> Result<VideoGenStatusResponse, SoraError> {
pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentialSet) -> Result<VideoGenStatusResponse, SoraError> {
let bearer_header = match credentials.jwt_bearer_token.as_ref() {
Some(bearer) => bearer.to_authorization_header_value(),
None => {
warn!("No JWT bearer for getting image gen status!");
return Err(SoraError::NoBearerTokenAvailable)
},
};
let client = reqwest::Client::new();
let mut url = reqwest::Url::parse(SORA_STATUS_URL)
@@ -216,8 +227,8 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &
let http_request = client.get(url)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie)
.header("Authorization", credentials.authorization_header_value())
.header("Cookie", credentials.cookies.as_str())
.header("Authorization", bearer_header)
.header("Content-Type", "application/json");
let http_request = http_request.build()?;
@@ -235,9 +246,15 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &
return Err(SoraError::AnyhowError(anyhow::anyhow!("The status request failed; status = {:?}, message = {:?}", status, response_body)));
}
let response: VideoGenStatusResponse = serde_json::from_str(response_body)?;
let maybe_response = serde_json::from_str::<VideoGenStatusResponse>(response_body);
Ok(response)
match maybe_response {
Ok(response) => Ok(response),
Err(err) => {
warn!("Failed to parse status response: {:?}", err);
Err(SoraError::JsonErrorWithBody(err, response_body.to_string()))
}
}
}
pub async fn save_generations_to_dir(generations: &[Generation], dir: &str) -> AnyhowResult<()> {
@@ -275,6 +292,7 @@ mod tests {
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
use crate::creds::sora_credential_set::SoraCredentialSet;
#[ignore]
#[tokio::test]
@@ -297,10 +315,12 @@ mod tests {
let bearer = bearer.trim().to_string();
let creds = SoraCredentials { bearer_token: bearer, cookie, sentinel: Some(sentinel) };
let new_creds = SoraCredentialSet::from_legacy_credentials(&creds)?;
// Get the task status for a specific task
// let response = get_image_gen_status(&StatusRequest::new(None, Some("task_01jr9yvpfyetx9r7qvvx38scna".to_string())), &creds).await?;
let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &creds).await?;
let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &new_creds).await?;
println!("task_id: {}", response.task_responses[0].id);
let task_id = response.task_responses[0].id.clone();
@@ -1,5 +1,5 @@
pub (crate) mod image_gen_http_request;
pub use image_gen_http_request::SoraError;
pub use image_gen_http_request::SoraImageGenError;
pub mod common;
pub mod sora_image_gen_remix;
pub mod sora_image_gen_simple;
@@ -1,7 +1,8 @@
use log::info;
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraError, VideoGenType};
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraImageGenError, VideoGenType};
use log::info;
use std::time::Duration;
pub struct SoraImageGenRemixRequest<'a> {
pub prompt: String,
@@ -9,11 +10,12 @@ pub struct SoraImageGenRemixRequest<'a> {
pub image_size: ImageSize,
pub sora_media_tokens: Vec<String>,
pub credentials: CredentialMigrationRef<'a>,
pub request_timeout: Option<Duration>,
}
/// The "remix" commands let you supply additional images as context.
/// Sora "media tokens" of previously uploaded images must be supplied.
pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Result<SoraImageGenResponse, SoraError> {
pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Result<SoraImageGenResponse, SoraImageGenError> {
let args = RawSoraImageGenRequest {
r#type: VideoGenType::ImageGen,
operation: OperationType::Remix,
@@ -39,7 +41,7 @@ pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Resu
// TODO: Replace this with debug level logging in the future.
info!("Sending remix request: {:?}", args);
let result = image_gen_http_request(args, request.credentials).await?;
let result = image_gen_http_request(args, request.credentials, request.request_timeout).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
@@ -80,6 +82,7 @@ mod tests {
image_size: ImageSize::Square,
sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()],
credentials: CredentialMigrationRef::Legacy(&creds),
request_timeout: None,
}).await?;
println!("task_id: {}", response.task_id);
@@ -1,14 +1,15 @@
use crate::credentials::SoraCredentials;
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType};
use errors::AnyhowResult;
use crate::creds::credential_migration::CredentialMigrationRef;
use std::time::Duration;
pub struct SoraImageGenSimpleRequest<'a> {
pub prompt: String,
pub num_images: NumImages,
pub image_size: ImageSize,
pub credentials: CredentialMigrationRef<'a>,
pub request_timeout: Option<Duration>,
}
pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
@@ -24,7 +25,7 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho
};
// TODO: Error handling.
let result = image_gen_http_request(sora_request, args.credentials).await?;
let result = image_gen_http_request(sora_request, args.credentials, args.request_timeout).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
@@ -34,12 +35,12 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho
#[cfg(test)]
mod tests {
use crate::credentials::SoraCredentials;
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::image_gen::common::{ImageSize, NumImages};
use crate::requests::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
use crate::creds::credential_migration::CredentialMigrationRef;
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
@@ -64,6 +65,7 @@ mod tests {
num_images: NumImages::One,
image_size: ImageSize::Square,
credentials: CredentialMigrationRef::Legacy(&creds),
request_timeout: None,
}).await?;
println!("task_id: {}", response.task_id);
@@ -2,10 +2,16 @@ use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
use crate::sora_error::SoraError;
use std::path::PathBuf;
use std::time::Duration;
/// Upload bytes.
/// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference.
pub async fn sora_media_upload_from_bytes(bytes: Vec<u8>, file_name: String, creds: CredentialMigrationRef<'_>) -> Result<SoraMediaUploadResponse, SoraError> {
pub async fn sora_media_upload_from_bytes(
bytes: Vec<u8>,
file_name: String,
creds: CredentialMigrationRef<'_>,
maybe_timeout: Option<Duration>,
) -> Result<SoraMediaUploadResponse, SoraError> {
let file_path = PathBuf::from(&file_name);
// TODO: Read file magic bytes first, then fall back to this.
@@ -20,5 +26,5 @@ pub async fn sora_media_upload_from_bytes(bytes: Vec<u8>, file_name: String, cre
_ => "application/octet-stream",
};
Ok(upload_media_http_request(bytes, file_name, mime_type, creds).await?)
Ok(upload_media_http_request(bytes, file_name, mime_type, creds, maybe_timeout).await?)
}
@@ -2,6 +2,7 @@ use crate::creds::credential_migration::CredentialMigrationRef;
use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
use crate::sora_error::SoraError;
use std::path::Path;
use std::time::Duration;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
@@ -9,7 +10,11 @@ use tokio::io::AsyncReadExt;
/// There's a better way to implement this.
const INITIAL_BUFFER_SIZE : usize = 1024*1024;
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: CredentialMigrationRef<'_>) -> Result<SoraMediaUploadResponse, SoraError> {
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(
file_path: P,
creds: CredentialMigrationRef<'_>,
maybe_timeout: Option<Duration>,
) -> Result<SoraMediaUploadResponse, SoraError> {
let mut file = File::open(&file_path).await?;
let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE);
file.read_to_end(&mut buffer).await?;
@@ -31,7 +36,7 @@ pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: Cr
_ => "application/octet-stream",
};
Ok(upload_media_http_request(buffer, filename, mime_type, creds).await?)
Ok(upload_media_http_request(buffer, filename, mime_type, creds, maybe_timeout).await?)
}
#[cfg(test)]
@@ -63,6 +68,7 @@ mod tests {
let response = sora_media_upload_from_file(
image_path,
CredentialMigrationRef::Legacy(&creds),
None,
).await?;
println!("media: {:?}", response);
@@ -1,3 +1,4 @@
use std::time::Duration;
use crate::credentials::{SoraCredentials, USER_AGENT};
use crate::creds::credential_migration::CredentialMigrationRef;
use crate::sora_error::SoraError;
@@ -69,6 +70,7 @@ pub (crate) async fn upload_media_http_request(
filename: String,
mime_type: &str,
credentials: CredentialMigrationRef<'_>,
maybe_timeout: Option<Duration>,
) -> Result<SoraMediaUploadResponse, SoraError> {
// Create multipart form
@@ -97,11 +99,15 @@ pub (crate) async fn upload_media_http_request(
// Make API request
let client = Client::new();
let request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
let mut request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
.multipart(form)
.header("User-Agent", USER_AGENT)
.header("Cookie", &cookie)
.header("Authorization", &auth_header);
if let Some(timeout) = maybe_timeout {
request_builder = request_builder.timeout(timeout);
}
let response = request_builder.send().await?;
@@ -29,6 +29,10 @@ pub enum SoraError {
/// Failed to generate a bearer token despite the `200 OK` response.
/// This has been generated by sending old/expired session cookies or empty cookies.
FailedToGenerateBearer,
/// Sora requires users to set up an account username before they can generate images.
/// This requires the user to set up that username.
SoraUsernameNotYetCreated,
/// Something is wrong with the JWT bearer token.
/// This error originates on our end as we try to parse the JWT.
@@ -58,6 +62,10 @@ pub enum SoraError {
/// serde_json::Error, likely from JSON deserialization schema mismatch.
JsonError(serde_json::Error),
/// serde_json::Error, likely from JSON deserialization schema mismatch.
/// Includes the original body.
JsonErrorWithBody(serde_json::Error, String),
}
impl Error for SoraError {}
@@ -71,6 +79,9 @@ impl Display for SoraError {
Self::FailedToGenerateBearer => {
write!(f, "Failed to generate bearer. Probably invalid session.")
}
Self::SoraUsernameNotYetCreated => {
write!(f, "Sora username not created. Please set up a username.")
}
Self::LocalJwtClaimsParseError(message) => {
write!(f, "Local JWT claims parse error: {}", message)
}
@@ -101,6 +112,9 @@ impl Display for SoraError {
Self::JsonError(err) => {
write!(f, "serde_json error: {}", err)
}
Self::JsonErrorWithBody(err, message) => {
write!(f, "serde_json error: {:?} body: {:?}", err, message)
}
}
}
}
@@ -40,7 +40,15 @@ Message: {
|| message.contains("token_expired");
if cookie_expired {
return SoraError::UnauthorizedCookieOrBearerExpired
return SoraError::UnauthorizedCookieOrBearerExpired;
}
let needs_onboarding =
message.contains("You must onboard before using this service")
|| message.contains("onboarding_required");
if needs_onboarding {
return SoraError::SoraUsernameNotYetCreated;
}
let status_code = status.as_u16();
@@ -31,7 +31,17 @@ impl <T: Clone> CloneSlot<T> {
Ok(store) => Ok(store.clone()),
}
}
pub fn get_clone_required(&self) -> AnyhowResult<T> {
match self.store.read() {
Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)),
Ok(store) => match &*store {
None => Err(anyhow!("Required stored item is not set")),
Some(inner) => Ok(inner.clone()),
}
}
}
pub fn set_clone(&self, value: &T) -> AnyhowResult<()> {
match self.store.write() {
Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)),
@@ -0,0 +1,29 @@
[package]
name = "file_to_base64_url"
edition = "2021"
version = "0.0.1"
publish = false
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
[lib]
name = "fal_client"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration = { workspace = true }
# Internal
errors = { path = "../../errors" }
# External
anyhow = { workspace = true }
base64 = { workspace = true }
mimetypes = { workspace = true }
[dev-dependencies]
testing = { path = "../../testing" }
filesys = { path = "../../files/filesys" }
@@ -0,0 +1,26 @@
use anyhow::anyhow;
use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE, BASE64_URL_SAFE_NO_PAD};
use base64::Engine;
use errors::AnyhowResult;
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use std::path::Path;
/// Read a file into a base64 URL-safe string.
pub fn file_to_base64_url<P: AsRef<Path>>(path: P) -> AnyhowResult<String> {
let bytes = std::fs::read(path.as_ref())?;
let mime = match MimetypeInfo::get_for_bytes(&bytes) {
Some(mime) => mime,
None => {
return Err(anyhow!("could not determine mimetype"));
},
};
let base64_bytes = BASE64_STANDARD.encode(&bytes);
match mime.mime_type() {
"image/png" => Ok(format!("data:image/png;base64,{}", base64_bytes)),
"image/jpeg" => Ok(format!("data:image/jpeg;base64,{}", base64_bytes)),
_ => Err(anyhow!("unknown or unsupported mime type: {}", mime.mime_type())),
}
}
@@ -0,0 +1 @@
pub mod file_to_base64_url;
@@ -15,7 +15,7 @@ use log::warn;
use log::{debug, error};
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use openai_sora_client::credentials::SoraCredentials;
use openai_sora_client::requests::image_gen::SoraError;
use openai_sora_client::requests::image_gen::SoraImageGenError;
use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token;
use serde::Deserialize;
use serde::Serialize;
@@ -326,7 +326,7 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest,
debug!("Sora image gen response: {:?}", response);
if let Err(SoraError::SentinelBlock(msg)) = &response {
if let Err(SoraImageGenError::SentinelBlock(msg)) = &response {
error!("Sora sentinel block, attempting refresh: {}", msg);
match generate_token().await {
@@ -382,16 +382,16 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest,
let response = response.map_err(|err| {
match err {
SoraError::TokenExpired(msg) => {
SoraImageGenError::TokenExpired(msg) => {
error!("Sora token expired, needs refresh: {}", msg);
// TODO: Implement token refresh logic here
EnqueueImageGenRequestError::ServerError
},
SoraError::SentinelBlock(msg) => {
SoraImageGenError::SentinelBlock(msg) => {
error!("Sora sentinel block still occurring after refresh: {}", msg);
EnqueueImageGenRequestError::ServerError
},
SoraError::TooManyConcurrentTasks(msg) => {
SoraImageGenError::TooManyConcurrentTasks(msg) => {
error!("Sora too many concurrent tasks: {}", msg);
EnqueueImageGenRequestError::TooManyConcurrentTasks
},
@@ -320,7 +320,7 @@ export const PromptBox = () => {
if (!prompt.trim()) return;
try {
console.log(
"Enqueuing with prompt:",
"(3) Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
+12 -3
View File
@@ -1,11 +1,20 @@
<!doctype html>
<html lang="en">
<head>
<title>ArtCraft</title>
<meta charset="utf-8" />
<title>ArtCraft 3D</title>
<base href="/" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-Y4Y8KMLPQ7"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-Y4Y8KMLPQ7');
</script>
<base href="/" />
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
@@ -1,4 +1,4 @@
import { useState } from "react";
import { useContext, useEffect, useState } from "react";
import {
faChevronLeft,
faGear,
@@ -18,6 +18,7 @@ import { TabSelector, TabItem } from "@storyteller/ui-tab-selector";
import { Signal } from "@preact/signals-react";
import { useSignals } from "@preact/signals-react/runtime";
import { setLogoutStates } from "~/signals/authentication/utilities";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
function isEditorPath(path: string) {
if (path === "/") return true;
@@ -29,14 +30,15 @@ interface Props {
pageName: string;
appTabIdSignal: Signal<string>;
setAppTabId: (id: string) => void;
is3DInitSignal: Signal<boolean>;
}
const appTabs: TabItem[] = [
{ id: "3D", label: "3D" },
{ id: "2D", label: "2D" },
{ id: "3D", label: "3D" },
];
export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
export const TopBar = ({ pageName, appTabIdSignal, setAppTabId, is3DInitSignal }: Props) => {
useSignals();
const currentLocation = getCurrentLocationWithoutParams(
@@ -47,6 +49,22 @@ export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
const [activeLibraryTab, setActiveLibraryTab] = useState("my-media");
const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
const engine3D = useContext(EngineContext);
const handleTabChange = (tabId: string) => {
if (appTabIdSignal.peek() == "3D") {
engine3D?.unmountEngine();
}
setAppTabId(tabId);
}
const currentAppTabId = appTabIdSignal.value;
const is3DInit = is3DInitSignal.value;
useEffect(() => {
if (currentAppTabId == "3D" && is3DInit) {
engine3D?.remountEngine();
}
}, [currentAppTabId, engine3D, is3DInit]);
return (
<>
<header className="fixed left-0 top-0 z-[60] w-full border-b border-white/5 bg-ui-background">
@@ -68,13 +86,13 @@ export const TopBar = ({ pageName, appTabIdSignal, setAppTabId }: Props) => {
Back to Editor
</ButtonLink>
)}
{/* <TabSelector
<TabSelector
tabs={appTabs}
activeTab={appTabIdSignal.value}
disabled={false}
onTabChange={(tabId: string) => setAppTabId(tabId)}
onTabChange={handleTabChange}
className="w-fit"
/> */}
/>
</div>
<div className="flex items-center justify-center gap-2 font-medium">
@@ -0,0 +1,36 @@
// This is used to do things such as disable job polling in the dev console.
export const SetDevFlag = (flagName: string, value: any) => {
if (!!!(window as any)._dev_flags) {
(window as any)._dev_flags = {};
}
(window as any)._dev_flags[flagName] = value;
}
export const GetDevFlag = (flagName: string) => {
if (!!(window as any)._dev_flags && flagName in (window as any)._dev_flags) {
return (window as any)._dev_flags[flagName];
}
return null;
}
export const DisableJobPolling = () => {
SetDevFlag("disable_job_polling", true);
}
export const EnableJobPolling = () => {
SetDevFlag("disable_job_polling", false);
}
export const IsJobPollingDisabled = () : boolean => {
return GetDevFlag("disable_job_polling") === true;
}
(window as any).setDevFlag = SetDevFlag;
(window as any).getDevFlag = GetDevFlag;
(window as any).disableJobPolling = DisableJobPolling;
(window as any).enableJobPolling = EnableJobPolling;
@@ -1,9 +1,15 @@
import { ToastTypes } from "~/enums";
import { addToast, setJobs } from "~/signals";
import { JobsApi } from "~/Classes/ApiManager/JobsApi";
import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags";
export async function PollRecentJobs() {
const jobsApi = new JobsApi();
if (IsJobPollingDisabled()) {
return;
}
const response = await jobsApi.ListRecentJobs();
if (response.success && response.data) {
setJobs(response.data);
@@ -28,6 +28,7 @@ import { invoke } from "@tauri-apps/api/core";
import { DecodeBase64ToImage } from "./utilities/DecodeBase64ToImage";
import { EncodeImageBitmapToBase64 } from "./utilities/EncodeImageBitmapToBase64";
import { FalBackgroundRemoval, FalBackgroundRemovalRequest } from "@storyteller/tauri-api";
@@ -443,6 +444,8 @@ export class Engine {
}
removeBackground() {
console.log("Remove background");
const selectedNodeSet = this.selectionManager.getSelectedNodes();
if (selectedNodeSet.size !== 1) {
@@ -460,15 +463,30 @@ export class Engine {
) => {
const image: HTMLImageElement = selectedNode.kNode.image();
const base64Bitmap = await EncodeImageBitmapToBase64(image);
const base64BitmapResponse = await invoke("remove_background", {
image: base64Bitmap,
});
const createdImage = await DecodeBase64ToImage(
base64BitmapResponse as string,
);
const falRemoveBgRequest: FalBackgroundRemovalRequest = {
base64_image: base64Bitmap,
};
callback(selectedNode, createdImage);
const response = await FalBackgroundRemoval(falRemoveBgRequest);
if (response.status !== "success") {
console.error("Failed to remove background", response);
// TODO
return;
}
console.log("Remove background success", response)
if ("payload" in response) {
const base64BitmapResponse = response.payload?.base64_bytes;
const createdImage = await DecodeBase64ToImage(
base64BitmapResponse as string,
);
callback(selectedNode, createdImage);
}
};
const _promise = removeBgInvocation(
@@ -57,6 +57,7 @@ import { BufferType, EngineFrameBuffers } from "./VideoProcessor/engine_buffer";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
import { is3DEditorInitialized, setIs3DEditorInitialized } from "~/signals/appTab";
export type EditorInitializeConfig = {
sceneToken: string;
@@ -197,6 +198,11 @@ class Editor {
processingHasFailed: boolean;
stats: Stats;
// Allows us to cancel the queued render
private renderEventToken: number;
private shouldRender: boolean;
private isMounted: boolean;
constructor() {
this.processingHasFailed = false;
console.log(
@@ -256,6 +262,8 @@ class Editor {
this.lastFrameTime = 0;
this.last_selected_sum = 0;
this.selectedCanvas = false;
this.renderEventToken = -1;
this.shouldRender = false;
// Audio Engine Test.
this.render_camera_aspect_ratio = CameraAspectRatio.HORIZONTAL_3_2;
@@ -595,13 +603,6 @@ class Editor {
this.cam_obj = this.activeScene.get_object_by_name(this.camera_name);
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
this.timeline.characters,
);
if (result) outlinerState.items.value = result.items;
}
this.mouse_controls = new MouseControls(
this.camera,
this.get_camera_person_mode.bind(this),
@@ -642,10 +643,19 @@ class Editor {
this.mouse_controls.sceneManager = this.sceneManager;
}
// Creates the main update loop.
//this.renderer.setAnimationLoop(this.updateLoop.bind(this));
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
this.timeline.characters,
);
if (result) outlinerState.items.value = result.items;
}
this.updateLoop();
// Attach freecam state controller
this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
// This will enable all event and render loops
// We'll disable it here so the UI events can control is manually
// this.remountEngine();
if (!this.utils.isEmpty(sceneToken)) {
this.loadScene(sceneToken);
@@ -658,24 +668,6 @@ class Editor {
});
}
document.addEventListener("mouseover", (event) => {
if (this.cameraViewControls) {
if (
event.target instanceof HTMLCanvasElement ||
(event.target as HTMLElement).id == "letterbox"
) {
this.cameraViewControls.enabled = true;
this.selectedCanvas = true;
this.focused = true;
} else {
this.cameraViewControls.reset();
this.focused = false;
this.cameraViewControls.enabled = false;
this.selectedCanvas = false;
}
}
});
this._configurePostProcessingRaw();
loadingBarData.value = {
@@ -683,6 +675,8 @@ class Editor {
progress: 100,
};
loadingBarIsShowing.value = false;
setIs3DEditorInitialized(true);
}
public isMovable(): boolean {
@@ -693,6 +687,32 @@ class Editor {
document.body.appendChild(this.stats.dom);
}
private mouseOverEventHandler(event: MouseEvent) {
if (this.cameraViewControls) {
if (
event.target instanceof HTMLCanvasElement ||
(event.target as HTMLElement).id == "letterbox"
) {
this.cameraViewControls.enabled = true;
this.selectedCanvas = true;
this.focused = true;
} else {
this.cameraViewControls.reset();
this.focused = false;
this.cameraViewControls.enabled = false;
this.selectedCanvas = false;
}
}
}
public enableFreeCamControls() {
document.addEventListener("mouseover", this.mouseOverEventHandler);
}
public disableFreeCamControls() {
document.removeEventListener("mouseover", this.mouseOverEventHandler);
}
// Captures the scene without the grid
public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
if (!this.renderer?.domElement || !this.camera) {
@@ -840,7 +860,6 @@ class Editor {
}
const uuid = crypto.randomUUID();
const file = new File([ab], `${uuid}.png`, { type: mimeString });
return { base64Snapshot, file };
}
@@ -1155,10 +1174,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 576
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
@@ -1167,10 +1186,10 @@ class Editor {
: this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
? 1024
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 900
: 1000;
@@ -1377,10 +1396,15 @@ class Editor {
}
// Basicly Unity 3D's update loop.
async updateLoop() {
updateLoop() {
if (!this.shouldRender) {
console.debug("Stopping 3D render loop");
return;
}
// Performance improvement: Handle frame cap
// Request the next render already - this is necessary so the loop doesn't stop if the fps cap is hit
requestAnimationFrame(this.updateLoop.bind(this));
this.renderEventToken = requestAnimationFrame(this.updateLoop.bind(this));
const frameTime = performance.now();
if (frameTime - this.lastFrameTime < 1000 / this.cap_fps) {
return;
@@ -1390,6 +1414,49 @@ class Editor {
this.renderSingleFrame();
}
startRenderLoop() {
if (this.shouldRender) {
console.warn("Render flag is already true");
return;
}
this.shouldRender = true;
this.updateLoop();
}
stopRenderLoop() {
this.shouldRender = false;
if (this.renderEventToken) {
cancelAnimationFrame(this.renderEventToken);
this.renderEventToken = -1;
}
}
remountEngine() {
if (!is3DEditorInitialized.peek()) {
console.log("3D mount: Wait for initialization");
return;
}
if (this.isMounted) {
console.log("3D already mounted, skipping");
return;
}
this.startRenderLoop();
this.sceneManager?.attachEventListeners();
this.enableFreeCamControls();
this.cameraViewControls?.attachEventListeners();
}
unmountEngine() {
this.stopRenderLoop();
this.sceneManager?.detachEventListeners();
this.disableFreeCamControls();
this.cameraViewControls?.detachEventListeners();
}
change_mode(type: "translate" | "rotate" | "scale") {
if (this.control == undefined) {
return;
@@ -10,13 +10,14 @@ import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox";
export const isSafariUserAgent = () => {
export const isSafariOrWebKit = () => {
const ua = navigator.userAgent;
const isWebKit = /AppleWebKit/i.test(ua) && !/Chrome|Chromium|Edg|OPR|CriOS|FxiOS/i.test(ua);
const isSafari = /^((?!chrome|android|crios|fxios).)*safari/i.test(ua);
return isSafari;
}
return isSafari || isWebKit;
};
export const hasApplePay = () => {
// @ts-expect-error Apple Pay is not defined in TypeScript
@@ -24,7 +25,7 @@ export const hasApplePay = () => {
}
export const isSafari = () => {
return isSafariUserAgent() || hasApplePay();
return isSafariOrWebKit() || hasApplePay();
}
export const isPointerLockSupported = () => {
@@ -83,6 +84,10 @@ class FreeCam extends EventDispatcher {
this.keydown = this.keydown.bind(this);
this.keyup = this.keyup.bind(this);
this.mousedown = this.mousedown.bind(this);
this.mouseup = this.mouseup.bind(this);
this.mousemove = this.mousemove.bind(this);
this.mousewheel = this.mousewheel.bind(this);
this.reset = this.reset.bind(this);
this.update = this.update.bind(this);
this.updateMovementVector = this.updateMovementVector.bind(this);
@@ -94,18 +99,28 @@ class FreeCam extends EventDispatcher {
event.preventDefault();
});
window.addEventListener("keydown", this.keydown.bind(this));
window.addEventListener("keyup", this.keyup.bind(this));
window.addEventListener("mousedown", this.mousedown.bind(this));
window.addEventListener("mouseup", this.mouseup.bind(this));
window.addEventListener("mousemove", this.mousemove.bind(this));
window.addEventListener("wheel", this.mousewheel.bind(this));
this.updateMovementVector();
this.updateRotationVector();
}
attachEventListeners() {
window.addEventListener("keydown", this.keydown);
window.addEventListener("keyup", this.keyup);
window.addEventListener("mousedown", this.mousedown);
window.addEventListener("mouseup", this.mouseup);
window.addEventListener("mousemove", this.mousemove);
window.addEventListener("wheel", this.mousewheel);
}
detachEventListeners() {
window.removeEventListener("keydown", this.keydown);
window.removeEventListener("keyup", this.keyup);
window.removeEventListener("mousedown", this.mousedown);
window.removeEventListener("mouseup", this.mouseup);
window.removeEventListener("mousemove", this.mousemove);
window.removeEventListener("wheel", this.mousewheel);
}
keydown(event: KeyboardEvent) {
if (!this.enabled || isPromptBoxFocused.value) {
return;
@@ -22,6 +22,7 @@ import {
poseMode,
showPoseControls,
} from "../signals/selectedMode";
import { Euler } from "three";
export enum KinMode {
FK,
@@ -65,6 +66,7 @@ export class MouseControls {
private fkHelper: FKHelper;
private isBoneDragged: boolean = false;
private ignoreNextClick: boolean = false;
private manualMouseLock = false;
constructor(
camera: THREE.PerspectiveCamera,
@@ -377,6 +379,60 @@ export class MouseControls {
}
}
handleMousePointerLock() {
if (this.isMouseClicked && this.lockControls) {
if (this.lockControls.isLocked == false) {
this.lockControls.lock();
}
} else if (this.lockControls) {
if (this.lockControls.isLocked == true) {
this.lockControls.unlock();
}
}
}
handleMouseManualLock(event: MouseEvent) {
if (this.isMouseClicked && this.lockControls) {
// If the mouse is clicked and the lockControls is not locked, lock it
if (this.lockControls.isLocked == false) {
// Lock the mouse flag
this.lockControls.isLocked = true;
// Change the cursor to a dragging cursor
this.lockControls.domElement.style.cursor = "move";
console.log("Mouse locked manually");
return;
}
// If the mouse is clicked but controls also locked, move the camera with the mouse
const camera = this.lockControls.getObject();
const _euler = new Euler(0, 0, 0, 'YXZ');
const _MOUSE_SENSITIVITY = 0.002;
const pointerSpeed = 1.0;
const _PI_2 = Math.PI / 2;
const minPolarAngle = 0;
const maxPolarAngle = Math.PI;
_euler.setFromQuaternion(camera.quaternion);
_euler.y -= event.movementX * _MOUSE_SENSITIVITY * pointerSpeed;
_euler.x -= event.movementY * _MOUSE_SENSITIVITY * pointerSpeed;
_euler.x = Math.max(_PI_2 - maxPolarAngle, Math.min(_PI_2 - minPolarAngle, _euler.x));
camera.quaternion.setFromEuler(_euler);
} else if (this.lockControls) {
if (this.lockControls.isLocked == true) {
// Unlock the mouse flag
this.lockControls.isLocked = false;
// Change the cursor back to default
this.lockControls.domElement.style.cursor = "default";
console.log("Mouse unlocked manually");
}
}
}
// Sets new mouse location usually used in raycasts.
onMouseMove(event: MouseEvent) {
if (this.canvReference == undefined) {
@@ -392,18 +448,11 @@ export class MouseControls {
// this causes an issue https://discourse.threejs.org/t/unable-to-use-pointer-lock-api/11092
if (!isPointerLockSupported()) {
return;
this.handleMouseManualLock(event);
} else {
this.handleMousePointerLock();
}
if (this.isMouseClicked && this.lockControls) {
if (this.lockControls.isLocked == false) {
this.lockControls.lock();
}
} else if (this.lockControls) {
if (this.lockControls.isLocked == true) {
this.lockControls.unlock();
}
}
}
// When the mouse clicks the screen.
@@ -65,6 +65,7 @@ export class SceneManager implements SceneManagerAPI {
private lastSceneState: SceneState;
private copiedObject: THREE.Object3D | undefined;
private is_character: Function;
private devMode: boolean;
constructor(
version: number,
@@ -80,14 +81,35 @@ export class SceneManager implements SceneManagerAPI {
this.lastSceneState = this.getSceneState();
this.updateOutliner = updateOutliner;
this.is_character = is_character;
this.devMode = devMode;
if (devMode) {
window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
window.addEventListener("click", this.onMouseClick.bind(this), false);
window.addEventListener("keydown", this.onKeyDown.bind(this), false);
window.addEventListener("mousedown", this.onMouseDown.bind(this), false);
window.addEventListener("mouseup", this.onMouseUp.bind(this), false);
this.onMouseMove = this.onMouseMove.bind(this);
this.onMouseClick = this.onMouseClick.bind(this);
this.onKeyDown = this.onKeyDown.bind(this);
this.onMouseDown = this.onMouseDown.bind(this);
this.onMouseUp = this.onMouseUp.bind(this);
}
public attachEventListeners() {
if (!this.devMode) {
return;
}
window.addEventListener("mousemove", this.onMouseMove, false);
window.addEventListener("click", this.onMouseClick, false);
window.addEventListener("keydown", this.onKeyDown, false);
window.addEventListener("mousedown", this.onMouseDown, false);
window.addEventListener("mouseup", this.onMouseUp, false);
}
public detachEventListeners() {
if (!this.devMode) {
return;
}
window.removeEventListener("mousemove", this.onMouseMove, false);
window.removeEventListener("click", this.onMouseClick, false);
window.removeEventListener("keydown", this.onKeyDown, false);
window.removeEventListener("mousedown", this.onMouseDown, false);
window.removeEventListener("mouseup", this.onMouseUp, false);
}
public async undo() {
@@ -121,7 +143,7 @@ export class SceneManager implements SceneManagerAPI {
name: string,
position: THREE.Vector3,
): Promise<THREE.Object3D<THREE.Object3DEventMap>> {
if (media_token.includes("SKY::")){
if (media_token.includes("SKY::")) {
const token = media_token.replace("SKY::", "");
this.scene.updateSkybox(token);
}
@@ -243,16 +265,16 @@ export class SceneManager implements SceneManagerAPI {
}
public async copy() {
// TODO MAKE BETTER FIX: Temp disables copy and paste of characters.
// TODO MAKE BETTER FIX: Temp disables copy and paste of characters.
const object = this.mouse_controls.selected?.at(0)
if(object !== undefined ){
if(this.is_character(object.uuid) === false)
this.copiedObject = object;
if (object !== undefined) {
if (this.is_character(object.uuid) === false)
this.copiedObject = object;
}
}
public async paste() {
if(this.copiedObject && this.copiedObject.name != "::CAM::") {
if (this.copiedObject && this.copiedObject.name != "::CAM::") {
const userdata = this.copiedObject.userData;
const position = this.copiedObject.position.clone();
const rotation = this.copiedObject.rotation.clone();
@@ -264,7 +286,7 @@ export class SceneManager implements SceneManagerAPI {
const obj = await this.create(media_id, name, position);
this.scene.setColor(obj.uuid, color);
obj.position.copy(position.add(new THREE.Vector3(0.5,0.0,0.5)));
obj.position.copy(position.add(new THREE.Vector3(0.5, 0.0, 0.5)));
obj.rotation.copy(rotation);
obj.scale.copy(scale);
@@ -7,23 +7,37 @@ export const InstallSounds = () => {
return;
}
// Good for simple immediate enqueue
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); // DEFAULT
// Good for simple menu choices
r.setSoundOnce("click", new SoundEffect("resources/sound/smrpg_click.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("scifi_menu_beep_1", new SoundEffect("resources/sound/metroidprime_UI_15.wav", { defaultVolume: 0.3 }));
r.setSoundOnce("scifi_menu_beep_2", new SoundEffect("resources/sound/metroidprime_UI_14.wav", { defaultVolume: 0.3 }));
r.setSoundOnce("scifi_menu_select", new SoundEffect("resources/sound/metroidprime_UI_18.wav", { defaultVolume: 0.3 }));
// Good for simple immediate enqueue success
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.4 })); // DEFAULT
// Good for simple immediate failure
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.4 }));
r.setSoundOnce("spike_throw", new SoundEffect("resources/sound/smrpg_enemy_spikethrow.wav", { defaultVolume: 0.1 })); // DEFAULT
r.setSoundOnce("giant_shell_kick", new SoundEffect("resources/sound/smrpg_mario_giantshellkick.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("wrong", new SoundEffect("resources/sound/smrpg_wrong.wav", { defaultVolume: 0.4 }));
// Good for simple async success
r.setSoundOnce("special_flower", new SoundEffect("resources/sound/smrpg_specialflower.wav", { defaultVolume: 0.2 })); // DEFAULT
r.setSoundOnce("extra_power", new SoundEffect("resources/sound/smrpg_character_extrapower.wav", { defaultVolume: 0.2 })); // DEFAULT
// Good for async errors
r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); // DEFAULT
r.setSoundOnce("ghost", new SoundEffect("resources/sound/smrpg_ghost.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("special_alert", new SoundEffect("resources/sound/goldensun_214.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("scifi_alert", new SoundEffect("resources/sound/metroidprime_UI_52.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("scifi_shrill_alert", new SoundEffect("resources/sound/metroidprime_UI_51.wav", { defaultVolume: 0.2 }));
// Good for menus
r.setSoundOnce("next", new SoundEffect("resources/sound/oot_dialogue_next.wav", { defaultVolume: 0.2 }));
r.setSoundOnce("select", new SoundEffect("resources/sound/goldensun_111.wav", { defaultVolume: 0.4 }));
r.setSoundOnce("scifi_menu_open", new SoundEffect("resources/sound/metroidprime_UI_12.wav", { defaultVolume: 0.3 }));
r.setSoundOnce("scifi_menu_close", new SoundEffect("resources/sound/metroidprime_UI_13.wav", { defaultVolume: 0.3 }));
// Good for special reward / celebration
r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.1 }));
@@ -28,7 +28,7 @@ import { PromptBox3D } from "@storyteller/ui-promptbox";
import { PopoverItem } from "@storyteller/ui-popover";
import { OnboardingHelper } from "./comps/OnboardingHelper";
import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDisplay";
import { appTabId, setAppTabId } from "~/signals/appTab";
import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
import { KonvaCanvasContainer } from "../Page2d/KonvaCanvasContainer";
import { KonvaRootComponent } from "../Page2d/KonvaRootComponent";
import {
@@ -251,13 +251,8 @@ export const PageEditor = () => {
});
};
const ContentFor2D = () => {
return (
<KonvaCanvasContainer>
<KonvaRootComponent className="h-full w-full" />
</KonvaCanvasContainer>
);
};
const display3d = appTabId.value === "3D" ? "block" : "none";
const display2d = appTabId.value === "2D" ? "block" : "none";
return (
<div className="w-screen">
@@ -265,108 +260,109 @@ export const PageEditor = () => {
pageName="Edit Scene"
appTabIdSignal={appTabId}
setAppTabId={setAppTabId}
is3DInitSignal={is3DEditorInitialized}
/>
{appTabId.value === "3D" ? (
<>
<OnboardingHelper />
<>
<OnboardingHelper />
<div
className="relative flex w-screen"
style={{ height: "calc(100vh - 68px)", display: display3d }}
>
{/* Engine section/side panel */}
<div
className="relative flex w-screen"
style={{ height: "calc(100vh - 68px)" }}
id="engine-n-panels-wrapper"
className="flex"
style={{
height,
}}
>
{/* Engine section/side panel */}
<div
id="engine-n-panels-wrapper"
className="flex"
style={{
height,
}}
>
<div className="relative w-full overflow-hidden bg-transparent">
<SceneContainer>
<EditorCanvas />
<PreviewFrameImage />
</SceneContainer>
<div className="relative w-full overflow-hidden bg-transparent">
<SceneContainer>
<EditorCanvas />
<PreviewFrameImage />
</SceneContainer>
{/* Focal Length Display */}
<FocalLengthDisplay />
{/* Focal Length Display */}
<FocalLengthDisplay />
{/* Pose Mode Selector */}
<PoseModeSelector />
{/* Pose Mode Selector */}
<PoseModeSelector />
{/* Top controls */}
<div
className="absolute left-0 top-0 w-full"
onClick={handleOverlayClick}
>
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
</div>
{/* Top controls */}
<div
className="absolute left-0 top-0 w-full"
onClick={handleOverlayClick}
>
<div className="grid grid-cols-3 gap-4">
<ControlsTopButtons />
<Controls3D />
</div>
{/* Bottom controls */}
<div
className="absolute bottom-0 left-0"
style={{
width: pageWidth.value,
}}
onClick={handleOverlayClick}
>
<div
className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
style={{ transform: `scale(${getScale()})` }}
>
<Outliner />
<PreviewEngineCamera />
</div>
<ControlPanelSceneObject />
</div>
<PromptBox3D
cameras={cameras}
cameraAspectRatio={cameraAspectRatio}
disableHotkeyInput={disableHotkeyInput}
enableHotkeyInput={enableHotkeyInput}
gridVisibility={gridVisibility}
setGridVisibility={setGridVisibility}
selectedCameraId={selectedCameraId}
deleteCamera={deleteCamera}
focalLengthDragging={focalLengthDragging}
isPromptBoxFocused={isPromptBoxFocused}
uploadImage={uploadImage}
handleCameraSelect={handleCameraSelect}
handleAddCamera={handleAddCamera}
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
setEnginePrompt={(prompt) => {
if (!editorEngine) {
return;
}
editorEngine!.positive_prompt = prompt;
}}
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame}
/>
<LoadingDots
className="absolute left-0 top-0 z-50"
isShowing={editorLoader.value.isShowing}
type="bricks"
message={editorLoader.value.message}
/>
</div>
{/* Bottom controls */}
<div
className="absolute bottom-0 left-0"
style={{
width: pageWidth.value,
}}
onClick={handleOverlayClick}
>
<div
className="absolute bottom-0 mb-4 ml-4 flex origin-bottom-left flex-col gap-2"
style={{ transform: `scale(${getScale()})` }}
>
<Outliner />
<PreviewEngineCamera />
</div>
<ControlPanelSceneObject />
</div>
<PromptBox3D
cameras={cameras}
cameraAspectRatio={cameraAspectRatio}
disableHotkeyInput={disableHotkeyInput}
enableHotkeyInput={enableHotkeyInput}
gridVisibility={gridVisibility}
setGridVisibility={setGridVisibility}
selectedCameraId={selectedCameraId}
deleteCamera={deleteCamera}
focalLengthDragging={focalLengthDragging}
isPromptBoxFocused={isPromptBoxFocused}
uploadImage={uploadImage}
handleCameraSelect={handleCameraSelect}
handleAddCamera={handleAddCamera}
handleCameraNameChange={handleCameraNameChange}
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
onAspectRatioSelect={onAspectRatioSelect}
setEnginePrompt={(prompt) => {
console.log("setEnginePrompt", prompt);
if (!editorEngine) {
console.log("editorEngine is not available");
return;
}
editorEngine!.positive_prompt = prompt;
}}
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
/>
<LoadingDots
className="absolute left-0 top-0 z-50"
isShowing={editorLoader.value.isShowing}
type="bricks"
message={editorLoader.value.message}
/>
</div>
</div>
</>
) : (
<ContentFor2D />
)}
</div>
</>
<KonvaCanvasContainer style={{ display: display2d }}>
<KonvaRootComponent className="h-full w-full" />
</KonvaCanvasContainer>
<LoginModal
onClose={() => {}}
onClose={() => { }}
videoSrc2D="/resources/videos/artcraft-canvas-demo.mp4"
videoSrc3D="/resources/videos/artcraft-3d-demo.mp4"
openAiLogo="/resources/images/openai-logo.png"
@@ -8,7 +8,6 @@ import { ErrorDialog, LoadingDots } from "~/components";
import { GenerateModals } from "~/pages/PageEnigma/comps/GenerateModals/GenerateModals";
import { Toaster } from "@storyteller/ui-toaster";
import { EditorLoadingBar } from "./comps/EditorLoadingBar";
// import { Wizard } from "~/pages/PageEnigma/Wizard/Wizard";
import { useSignals } from "@preact/signals-react/runtime";
import { useEffect, useState } from "react";
import * as gpu from "detect-gpu";
@@ -23,8 +22,8 @@ import {
showPrecisionSelector,
} from "./signals/precisionSelectorMenu";
import { InstallSounds } from "~/pages/PageEnigma/InstallSounds";
import { InstallImageGenerationFailure } from "~/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure";
import { InstallImageGenerationSuccess } from "~/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess";
import { useImageGenerationFailureEvent } from "./TauriEvents/useImageGenerationFailureEvent";
import { useImageGenerationSuccessEvent } from "./TauriEvents/useImageGenerationSuccessEvent";
export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
useSignals();
@@ -90,11 +89,12 @@ export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
});
});
useImageGenerationSuccessEvent();
useImageGenerationFailureEvent();
useEffect(() => {
console.log("installing event listeners");
InstallSounds();
InstallImageGenerationSuccess();
InstallImageGenerationFailure();
});
if (validGpu === "unknown") {
@@ -18,7 +18,7 @@ import { StyleSelection } from "~/pages/PageEnigma/comps/StyleSelection";
import { StyleMoreOptions } from "~/pages/PageEnigma/comps/StyleMoreOptions/StyleMoreOptions";
import Queue, { QueueNames } from "~/pages/PageEnigma/Queue";
import { toTimelineActions } from "~/pages/PageEnigma/Queue/toTimelineActions";
import { appTabId, setAppTabId } from "~/signals/appTab";
import { appTabId, is3DEditorInitialized, setAppTabId } from "~/signals/appTab";
export const PageStyling = () => {
useSignals();
@@ -64,6 +64,7 @@ export const PageStyling = () => {
pageName="Stylization"
appTabIdSignal={appTabId}
setAppTabId={setAppTabId}
is3DInitSignal={is3DEditorInitialized}
/>
<div className="mt-4 flex flex-col items-center gap-6">
<PreviewImages />
@@ -1,24 +0,0 @@
import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
import { listen } from '@tauri-apps/api/event';
import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type SoraImageGenerationFailed = {
prompt: string,
};
export const InstallImageGenerationFailure = () => {
listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.error("Image generation failed!");
});
}
export const ImageGenerationFailure = () => {
}
@@ -1,23 +0,0 @@
import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
import { listen } from '@tauri-apps/api/event';
import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type ImageGenerationSuccess = {
media_file_token: string,
};
export const InstallImageGenerationSuccess = () => {
listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image generated!");
});
}
export const ImageGenerationSuccess = () => {
}
@@ -0,0 +1,41 @@
import { useEffect } from "react";
import { listen, UnlistenFn } from '@tauri-apps/api/event';
import { SoundRegistry } from "@storyteller/soundboard";
import { GetAppPreferences } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type SoraImageGenerationFailed = {
prompt: string,
};
export const useImageGenerationFailureEvent = () => {
useEffect(() => {
let isUnmounted = false;
let unlisten: Promise<UnlistenFn>;
const setup = async () => {
unlisten = listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
console.log("Image generation failure event received:", event);
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.error("Image generation failed!");
});
if (isUnmounted) {
unlisten.then(f => f()); // Unsubscribe if unmounted early.
}
};
setup();
return () => {
isUnmounted = true;
unlisten.then(f => f());
};
}, []);
}
@@ -0,0 +1,41 @@
import { useEffect } from "react";
import { listen, UnlistenFn } from '@tauri-apps/api/event';
import { SoundRegistry } from "@storyteller/soundboard";
import { GetAppPreferences } from "@storyteller/tauri-api";
import { toast } from "@storyteller/ui-toaster";
type ImageGenerationSuccess = {
media_file_token: string,
};
export const useImageGenerationSuccessEvent = () => {
useEffect(() => {
let isUnmounted = false;
let unlisten: Promise<UnlistenFn>;
const setup = async () => {
unlisten = listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
console.log("Image generation success event received:", event);
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.generation_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image generated!");
});
if (isUnmounted) {
unlisten.then(f => f()); // Unsubscribe if unmounted early.
}
};
setup();
return () => {
isUnmounted = true;
unlisten.then(f => f());
};
}, []);
}
@@ -9,6 +9,7 @@ import { useEffect, useState, useRef, useCallback } from "react";
import { JobsApi } from "~/Classes/ApiManager/JobsApi";
import { toast } from "@storyteller/ui-toaster";
import { ActiveJob } from "~/pages/PageEnigma/models";
import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags";
// TODO ensure we de-dupe all this extra code.
interface CompletedItem {
token: string;
@@ -85,6 +86,9 @@ export function Activity() {
let mounted = true;
async function fetchJobs() {
if (IsJobPollingDisabled()) {
return;
}
try {
const jobsApi = new JobsApi();
const response = await jobsApi.ListRecentJobs();
@@ -1,936 +0,0 @@
import { useState, useRef, useEffect, useContext } from "react";
import { useSignals } from "@preact/signals-react/runtime";
import { invoke } from "@tauri-apps/api/core";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
import { IsDesktopApp } from "@storyteller/tauri-utils";
import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
import { downloadFileFromUrl } from "@storyteller/api";
import {
faPlus,
faCamera,
faSave,
faSparkles,
faUpload,
faImages,
faDownload,
faSpinnerThird,
faTimes,
faMessageCheck,
faMessageXmark,
} from "@fortawesome/pro-solid-svg-icons";
import {
faRectangleWide,
faRectangleVertical,
faSquare,
faRectangle,
faTableCellsLarge,
} from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
import { Button, ToggleButton, Tooltip } from "~/components";
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
import {
cameraAspectRatio,
disableHotkeyInput,
DomLevels,
enableHotkeyInput,
gridVisibility,
setGridVisibility,
} from "~/pages/PageEnigma/signals";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
import { PromptsApi } from "@storyteller/api";
import { SoundRegistry } from "@storyteller/soundboard";
import { GetAppPreferences, SoraImageRemix } from "@storyteller/tauri-api";
import { CommandSuccessStatus } from "@storyteller/tauri-api";
import {
cameras,
selectedCameraId,
addCamera,
deleteCamera,
updateCamera,
focalLengthDragging,
} from "~/pages/PageEnigma/signals/camera";
import { toast } from "@storyteller/ui-toaster";
import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox";
interface ReferenceImage {
id: string;
url: string;
file: File;
mediaToken: string;
}
import { EngineApi } from "~/Classes/ApiManager/EngineApi";
import { UploaderStates } from "~/enums/UploaderStates";
import { CameraSettingsModal } from "../CameraSettingsModal";
import { twMerge } from "tailwind-merge";
import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal";
import { Modal } from "@storyteller/ui-modal";
export const PromptBox = () => {
useSignals();
const editorEngine = useContext(EngineContext);
const fileInputRef = useRef<HTMLInputElement>(null);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const [content, setContent] = useState<React.ReactNode>(null);
const [isModalOpen, setIsModalOpen] = useState(false);
const [prompt, setPrompt] = useState("");
const [isEnqueueing, setisEnqueueing] = useState(false);
const [useSystemPrompt, setUseSystemPrompt] = useState(true);
const [referenceImages, setReferenceImages] = useState<ReferenceImage[]>([]);
const [uploadingImages, setUploadingImages] = useState<
{ id: string; file: File }[]
>([]);
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
{
label: "3:2",
selected: true,
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
},
{
label: "2:3",
selected: false,
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
},
{
label: "1:1",
selected: false,
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
},
]);
const [isCameraSettingsOpen, setIsCameraSettingsOpen] = useState(false);
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[],
);
const [activeGalleryTab, setActiveGalleryTab] = useState("my-media");
useEffect(() => {
if (textareaRef.current) {
textareaRef.current.style.height = "auto";
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
}
}, [prompt]);
// Update aspect ratio list based on the current cameraAspectRatio signal
useEffect(() => {
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected:
(item.label === "16:9" &&
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9) ||
(item.label === "3:2" &&
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2) ||
(item.label === "2:3" &&
cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3) ||
(item.label === "1:1" &&
cameraAspectRatio.value === CameraAspectRatio.SQUARE_1_1),
})),
);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [cameraAspectRatio.value]);
const handleCameraSelect = (selectedItem: PopoverItem) => {
const selectedCamera = cameras.value.find(
(cam) => cam.label === selectedItem.label,
);
if (selectedCamera && editorEngine) {
selectedCameraId.value = selectedCamera.id;
// Show focal length display temporarily
// TODO: Rename dragging to visible - BFlat
focalLengthDragging.value = {
isDragging: true,
focalLength: selectedCamera.focalLength,
};
setTimeout(() => {
focalLengthDragging.value = {
isDragging: false,
focalLength: selectedCamera.focalLength,
};
}, 1500);
// Update the main camera to match the selected camera's properties
if (editorEngine.camera) {
// First update position and lookAt
editorEngine.camera.position.set(
selectedCamera.position.x,
selectedCamera.position.y,
selectedCamera.position.z,
);
editorEngine.camera.lookAt(
selectedCamera.lookAt.x,
selectedCamera.lookAt.y,
selectedCamera.lookAt.z,
);
// Update FOV
editorEngine.camera.fov = editorEngine.focalLengthToFov(
selectedCamera.focalLength,
);
editorEngine.camera.updateProjectionMatrix();
// Reset and update camera controls
if (editorEngine.cameraViewControls) {
editorEngine.cameraViewControls.reset();
editorEngine.cameraViewControls.update(0);
}
// Force a render to update the view
editorEngine.renderScene();
// Queue an update to ensure the engine processes the camera change
// Queue.publish({
// queueName: QueueNames.TO_ENGINE,
// action: toEngineActions.CAMERA_CHANGED,
// data: selectedCamera,
// });
}
// Force update camera properties in the state
updateCamera(selectedCamera.id, {
focalLength: selectedCamera.focalLength,
position: selectedCamera.position,
rotation: selectedCamera.rotation,
lookAt: selectedCamera.lookAt,
});
}
};
const handleAddCamera = () => {
// Check if we've reached the maximum number of cameras
if (cameras.value.length >= 6) {
console.warn("Maximum number of cameras (6) reached");
return;
}
const newIndex = cameras.value.length + 1;
const newId = `cam${newIndex}`;
// This is for generating random orbital position for the new camera using spherical coordinates
const radius = Math.random() * 5 + 7; // Distance from center: 7 to 12 units
const theta = Math.random() * Math.PI * 2; // Azimuthal angle: 0 to 2π
const phi = Math.PI / 3 + (Math.random() * Math.PI) / 6; // Polar angle: π/3 to π/2 (60° to 90° from horizontal)
// Convert spherical coordinates to Cartesian coordinates
const randomX = radius * Math.sin(phi) * Math.cos(theta);
const randomY = Math.abs(radius * Math.cos(phi)) + 2; // Ensure Y is positive and at least 2 units up
const randomZ = radius * Math.sin(phi) * Math.sin(theta);
addCamera({
id: newId,
label: `Camera ${newIndex}`,
focalLength: 24,
position: {
x: randomX,
y: randomY,
z: randomZ,
},
rotation: { x: 0, y: 0, z: 0 },
lookAt: { x: 0, y: 0, z: 0 },
});
// Switch to the newly created camera
selectedCameraId.value = newId;
// Update the engine camera to match the new camera's properties
if (editorEngine && editorEngine.camera) {
editorEngine.camera.position.set(randomX, randomY, randomZ);
editorEngine.camera.lookAt(0, 0, 0);
editorEngine.camera.fov = editorEngine.focalLengthToFov(24);
editorEngine.camera.updateProjectionMatrix();
// Reset and update camera controls
if (editorEngine.cameraViewControls) {
editorEngine.cameraViewControls.reset();
editorEngine.cameraViewControls.update(0);
}
// Force a render to update the view
editorEngine.renderScene();
// Queue.publish({
// queueName: QueueNames.TO_ENGINE,
// action: toEngineActions.CAMERA_CHANGED,
// data: {
// id: newId,
// label: `Camera ${newIndex}`,
// focalLength: 24,
// position: { x: randomX, y: randomY, z: randomZ },
// rotation: { x: 0, y: 0, z: 0 },
// lookAt: { x: 0, y: 0, z: 0 },
// },
// });
}
};
const handleCameraNameChange = (id: string, newName: string) => {
updateCamera(id, { label: newName });
};
const handleCameraFocalLengthChange = (id: string, value: number) => {
const camera = cameras.value.find((cam) => cam.id === id);
if (camera) {
updateCamera(id, { focalLength: value });
}
};
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatioList((prev) =>
prev.map((item) => ({
...item,
selected: item.label === selectedItem.label,
})),
);
// Map the selected label to the corresponding CameraAspectRatio enum value
let newRatio: CameraAspectRatio;
switch (selectedItem.label) {
case "16:9":
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
break;
case "3:2":
newRatio = CameraAspectRatio.HORIZONTAL_3_2;
break;
case "2:3":
newRatio = CameraAspectRatio.VERTICAL_2_3;
break;
case "1:1":
newRatio = CameraAspectRatio.SQUARE_1_1;
break;
default:
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
}
// Publish the change to the engine
Queue.publish({
queueName: QueueNames.TO_ENGINE,
action: toEngineActions.CHANGE_CAMERA_ASPECT_RATIO,
data: newRatio,
});
};
const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
const files = event.target.files;
if (files) {
Array.from(files).forEach((file) => {
const uploadId = Math.random().toString(36).substring(7);
setUploadingImages((prev) => [...prev, { id: uploadId, file }]);
const reader = new FileReader();
reader.onloadend = () => {
uploadImage({
title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
assetFile: file,
progressCallback: (newState) => {
console.debug("Upload progress:", newState.data);
if (newState.status === UploaderStates.success && newState.data) {
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: reader.result as string,
file,
mediaToken: newState.data || "",
};
setReferenceImages((prev) => [...prev, referenceImage]);
setUploadingImages((prev) =>
prev.filter((img) => img.id !== uploadId),
);
console.log("Reference image added:", referenceImage);
} else if (
newState.status === UploaderStates.assetError ||
newState.status === UploaderStates.imageCreateError
) {
setUploadingImages((prev) =>
prev.filter((img) => img.id !== uploadId),
);
console.error("Upload failed");
}
},
});
};
reader.readAsDataURL(file);
});
}
};
const handleRemoveReference = (id: string) => {
setReferenceImages((prev) => prev.filter((img) => img.id !== id));
if (fileInputRef.current) {
fileInputRef.current.value = "";
}
};
const handleUploadClick = () => {
fileInputRef.current?.click();
};
const handleGallerySelect = () => {
setIsGalleryModalOpen(true);
};
const handleGalleryClose = () => {
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
};
const handleImageSelect = (id: string) => {
setSelectedGalleryImages((prev) => {
if (prev.includes(id)) {
return prev.filter((imageId) => imageId !== id);
}
if (prev.length >= 4) {
return prev;
}
return [...prev, id];
});
};
const handleGalleryImages = (selectedItems: GalleryItem[]) => {
selectedItems.forEach((item) => {
if (!item.fullImage) return;
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: item.fullImage,
file: new File([], "gallery-image"),
mediaToken: item.id,
};
setReferenceImages((prev) => [...prev, referenceImage]);
});
setIsGalleryModalOpen(false);
setSelectedGalleryImages([]);
};
const handleAction = (action: string) => {
switch (action) {
case "upload":
handleUploadClick();
break;
case "gallery":
handleGallerySelect();
break;
default:
console.log("Unknown action:", action);
}
};
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
e.preventDefault();
const pastedText = e.clipboardData.getData("text").trim();
setPrompt(pastedText);
};
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
setPrompt(e.target.value);
};
const handleError = (errorMessage: string) => {
setContent(errorMessage);
setIsModalOpen(true);
};
const handleEnqueue = async () => {
const isDesktop = IsDesktopApp();
console.log("Is this a desktop app?", isDesktop);
const r = SoundRegistry.getInstance();
r.playSound("test");
if (isDesktop) {
await handleTauriEnqueue();
} else {
await handleWebEnqueue();
}
};
const handleWebEnqueue = async () => {
if (!prompt.trim()) return;
setisEnqueueing(true);
if (editorEngine) {
editorEngine.positive_prompt = prompt;
//const engineApi = new EngineApi();
const promptsApi = new PromptsApi();
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const snapshotResult = await promptsApi.uploadSceneSnapshot({
screenshot: snapshot.file,
sceneMediaToken: "",
});
console.log("useSystemPrompt", useSystemPrompt);
const response = await promptsApi.enqueueImageGeneration({
disableSystemPrompt: !useSystemPrompt,
prompt: prompt,
snapshotMediaToken: snapshotResult.data || "",
additionalImages: referenceImages.map((image) => image.mediaToken),
});
console.log("response", response);
if (response.errorMessage) {
handleError(response.errorMessage);
setisEnqueueing(false);
return;
}
}
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
);
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
setisEnqueueing(false);
toast.success("Image added to queue");
}
}
setisEnqueueing(false);
};
const handleTauriEnqueue = async () => {
if (!prompt.trim()) return;
setisEnqueueing(true);
setTimeout(() => {
// TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if
// the backend hangs on a given request. (Sometimes Sora doesn't respond.) Ideally what we would do instead is only
// force the prompt box to be enabled again as long as another request isn't running, ie. if we just fired off two
// requests in succession, we wouldn't want to turn it off as the first one gets submitted.
console.debug('Turn off blocking of prompt box...');
setisEnqueueing(false);
}, 10000);
if (editorEngine) {
editorEngine.positive_prompt = prompt;
const promptsApi = new PromptsApi();
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const snapshotResult = await promptsApi.uploadSceneSnapshot({
screenshot: snapshot.file,
//sceneMediaToken: "",
});
console.log("snapshotResult", snapshotResult)
const generateResponse = await SoraImageRemix({
snapshot_media_token: snapshotResult.data!,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken,
),
maybe_number_of_samples: 1,
});
console.log("generateResponse", generateResponse)
if (generateResponse.status === CommandSuccessStatus.Success) {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image added to queue");
setisEnqueueing(false);
return;
} else {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.error("Failed to enqueue image");
setisEnqueueing(false);
return;
}
// const generateResponse = await invoke("sora_image_remix_command", {
// request: {
// snapshot_media_token: snapshotResult.data!,
// disable_system_prompt: !useSystemPrompt,
// prompt: prompt,
// maybe_additional_images: referenceImages.map(
// (image) => image.mediaToken,
// ),
// maybe_number_of_samples: 1,
// },
// });
//if (generateResponse.errorMessage) {
// handleError(response.errorMessage);
// setisEnqueueing(false);
// return;
//}
}
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
);
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
setisEnqueueing(false);
toast.success("Image added to queue");
}
}
setisEnqueueing(false);
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleWebEnqueue();
}
};
// Get the current aspect ratio icon based on the cameraAspectRatio signal
const getCurrentAspectRatioIcon = () => {
switch (cameraAspectRatio.value) {
case CameraAspectRatio.HORIZONTAL_3_2:
return faRectangle;
case CameraAspectRatio.VERTICAL_2_3:
return faRectangleVertical;
case CameraAspectRatio.VERTICAL_9_16:
return faRectangleVertical;
case CameraAspectRatio.SQUARE_1_1:
return faSquare;
default:
return faRectangleWide;
}
};
const handleSaveFrame = async () => {
if (editorEngine) {
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const engineApi = new EngineApi();
await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file || "",
sceneMediaToken: "",
});
}
}
};
const handleDownloadFrame = () => {
if (editorEngine) {
editorEngine.snapShotOfCurrentFrame(true);
}
};
return (
<>
<Modal
isOpen={isModalOpen}
onClose={() => {
setIsModalOpen(false);
setContent("");
}}
>
{content}
</Modal>
<div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 flex-col gap-3">
{(referenceImages.length > 0 || uploadingImages.length > 0) && (
<div className="flex w-full gap-2">
{referenceImages.map((image) => (
<div
key={image.id}
className="glass relative aspect-square w-20 rounded-lg"
>
<img
src={image.url}
alt="Reference"
className="h-full w-full rounded-lg object-cover"
/>
<button
onClick={() => handleRemoveReference(image.id)}
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/40 text-white backdrop-blur-md transition-colors hover:bg-black"
>
<FontAwesomeIcon icon={faTimes} className="h-2.5 w-2.5" />
</button>
</div>
))}
{uploadingImages.map(({ id, file }) => {
const previewUrl = URL.createObjectURL(file);
return (
<div
key={id}
className="glass relative aspect-square w-20 overflow-hidden rounded-lg"
>
<div className="absolute inset-0">
<img
src={previewUrl}
alt="Uploading preview"
className="h-full w-full object-cover blur-sm"
/>
</div>
<div className="absolute inset-0 flex items-center justify-center bg-black/20">
<FontAwesomeIcon
icon={faSpinnerThird}
className="h-6 w-6 animate-spin text-white"
/>
</div>
</div>
);
})}
</div>
)}
<div
className={twMerge(
"glass w-[730px] rounded-xl p-4",
isPromptBoxFocused.value ? "!border !border-primary" : "",
)}
>
<input
type="file"
ref={fileInputRef}
className="hidden"
accept="image/*"
onChange={handleFileUpload}
multiple
/>
<div className="flex justify-center gap-2">
<PopoverMenu
mode="button"
panelTitle="Add Image"
items={[
{
label: "Upload from device",
selected: false,
icon: <FontAwesomeIcon icon={faUpload} className="h-4 w-4" />,
action: "upload",
},
{
label: "Choose from library",
selected: false,
icon: <FontAwesomeIcon icon={faImages} className="h-4 w-4" />,
action: "gallery",
},
]}
onPanelAction={handleAction}
showIconsInList
buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none"
triggerIcon={
<FontAwesomeIcon icon={faPlus} className="text-xl" />
}
/>
<textarea
ref={textareaRef}
rows={1}
placeholder="Describe your image..."
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
value={prompt}
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => {
disableHotkeyInput(DomLevels.INPUT);
isPromptBoxFocused.value = true;
}}
onBlur={() => {
enableHotkeyInput(DomLevels.INPUT);
isPromptBoxFocused.value = false;
}}
/>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<Tooltip
content="Aspect ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleAspectRatioSelect}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentAspectRatioIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
<Tooltip
content="Camera"
position="top"
className="z-50"
delay={300}
closeOnClick={true}
>
<PopoverMenu
items={cameras.value.map((cam) => ({
id: cam.id,
label: cam.label,
selected: cam.id === selectedCameraId.value,
icon: (
<FontAwesomeIcon icon={faCamera} className="h-4 w-4" />
),
focalLength: cam.focalLength,
position: cam.position,
rotation: cam.rotation,
lookAt: cam.lookAt,
}))}
onSelect={handleCameraSelect}
onAdd={handleAddCamera}
triggerIcon={
<FontAwesomeIcon icon={faCamera} className="h-4 w-4" />
}
showAddButton
disableAddButton={cameras.value.length >= 6}
showIconsInList
mode="toggle"
panelTitle="Camera"
panelActionLabel="Settings"
onPanelAction={() => setIsCameraSettingsOpen(true)}
buttonClassName="max-w-[130px] h-9"
/>
</Tooltip>
<Tooltip
content={
useSystemPrompt
? "Use system prompt: ON"
: "Use system prompt: OFF"
}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={useSystemPrompt}
icon={faMessageXmark}
activeIcon={faMessageCheck}
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
/>
</Tooltip>
<Tooltip
content={gridVisibility.value ? "Grid: ON" : "Grid: OFF"}
position="top"
className="z-50"
delay={200}
>
<ToggleButton
isActive={gridVisibility.value}
icon={faTableCellsLarge}
activeIcon={faTableCellsLarge}
onClick={() => setGridVisibility(!gridVisibility.value)}
/>
</Tooltip>
</div>
<div className="flex items-center gap-2">
<Tooltip
content="Download frame"
position="top"
className="z-50"
closeOnClick={true}
delay={200}
>
<Button
className="flex h-9 items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faDownload}
onClick={handleDownloadFrame}
/>
</Tooltip>
<Button
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faSave}
onClick={handleSaveFrame}
>
Save frame
</Button>
<Button
className="flex items-center border-none bg-brand-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
icon={!isEnqueueing ? faSparkles : undefined}
onClick={handleEnqueue}
disabled={isEnqueueing || !prompt.trim()}
>
{isEnqueueing ? (
<FontAwesomeIcon
icon={faSpinnerThird}
className="animate-spin text-lg"
/>
) : (
"Generate"
)}
</Button>
</div>
</div>
</div>
<CameraSettingsModal
isOpen={isCameraSettingsOpen}
onClose={() => setIsCameraSettingsOpen(false)}
cameras={cameras.value.map((cam) => ({
id: cam.id,
label: cam.label,
selected: cam.id === selectedCameraId.value,
icon: <FontAwesomeIcon icon={faCamera} className="h-4 w-4" />,
focalLength: cam.focalLength,
position: cam.position,
rotation: cam.rotation,
lookAt: cam.lookAt,
}))}
onCameraNameChange={handleCameraNameChange}
onCameraFocalLengthChange={handleCameraFocalLengthChange}
onAddCamera={handleAddCamera}
selectedCameraId={selectedCameraId.value}
handleCameraSelect={handleCameraSelect}
onDeleteCamera={deleteCamera}
/>
</div>
<GalleryModal
isOpen={isGalleryModalOpen}
onClose={handleGalleryClose}
mode="select"
selectedItemIds={selectedGalleryImages}
onSelectItem={handleImageSelect}
maxSelections={4}
onUseSelected={handleGalleryImages}
tabs={[
{ id: "my-media", label: "My generations" },
{ id: "uploads", label: "My uploads" },
]}
activeTab={activeGalleryTab}
onTabChange={setActiveGalleryTab}
onDownloadClicked={downloadFileFromUrl}
/>
</>
);
};
@@ -1 +0,0 @@
export * from "./PromptBox";
@@ -10,3 +10,8 @@ export const setAppTabId = (newId: string) => {
appTabId.value = newId;
};
export const is3DEditorInitialized = signal(false);
export const setIs3DEditorInitialized = (isInitialized: boolean) => {
is3DEditorInitialized.value = isInitialized;
}
+3 -11
View File
@@ -2,9 +2,9 @@
"extends": "../../tsconfig.base.json",
"include": [
"env.d.ts",
"**/*.js",
"**/*.ts",
"**/*.tsx",
"app/**/*.js",
"app/**/*.ts",
"app/**/*.tsx",
"netlify-edge-plugin.ts"
],
"compilerOptions": {
@@ -24,11 +24,6 @@
"allowJs": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"paths": {
"~/*": [
"apps/editor3d/app/src/*"
]
},
// Vite takes care of building everything, not tsc.
"noEmit": true
},
@@ -48,9 +43,6 @@
{
"path": "../../libs/components/toaster"
},
{
"path": "../../libs/components/demo-modal"
},
{
"path": "../../libs/components/login-modal"
},
@@ -1,7 +1,7 @@
import { useState, useRef, useEffect } from "react";
import { invoke } from "@tauri-apps/api/core";
import { useSignals } from "@preact/signals-react/runtime";
import { toast } from "sonner";
import { toast } from "@storyteller/ui-toaster";
import {
JobContextType,
MaybeCanvasRenderBitmapType,
@@ -32,6 +32,8 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { IsDesktopApp } from "@storyteller/tauri-utils";
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
import { PromptsApi } from "@storyteller/api";
import { CommandSuccessStatus, GetAppPreferences, SoraImageRemix, SoraImageRemixErrorType } from "@storyteller/tauri-api";
import { SoundRegistry } from "@storyteller/soundboard";
interface ReferenceImage {
id: string;
@@ -110,7 +112,7 @@ export const PromptBox2D = ({
textareaRef.current.style.height = "auto";
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
}
}, [prompt]);
});
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
setAspectRatioList((prev) =>
@@ -271,19 +273,73 @@ export const PromptBox2D = ({
console.log("useSystemPrompt", useSystemPrompt);
console.log("Snapshot media token:", snapshotMediaToken.data);
const generateResponse = await invoke("sora_image_remix_command", {
request: {
snapshot_media_token: snapshotMediaToken.data,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken
),
maybe_number_of_samples: 1,
},
//const generateResponse = await invoke("sora_image_remix_command", {
// request: {
// snapshot_media_token: snapshotMediaToken.data,
// disable_system_prompt: !useSystemPrompt,
// prompt: prompt,
// maybe_additional_images: referenceImages.map(
// (image) => image.mediaToken
// ),
// maybe_number_of_samples: 1,
// },
//});
//console.log("Generate response:", generateResponse);
//toast.success("Please wait while we process your image.");
const generateResponse = await SoraImageRemix({
snapshot_media_token: snapshotMediaToken.data,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken,
),
maybe_number_of_samples: 1,
});
console.log("Generate response:", generateResponse);
toast.success("Please wait while we process your image.");
console.log("generateResponse", generateResponse);
if (generateResponse.status === CommandSuccessStatus.Success) {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image added to queue");
} else {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
let errorMessage = "Failed to enqueue image";
if ("error_type" in generateResponse) {
switch (generateResponse.error_type) {
case SoraImageRemixErrorType.SoraLoginRequired:
errorMessage = "You need to log into Sora to continue. See the settings menu.";
break;
case SoraImageRemixErrorType.ServerError:
errorMessage = "Server error. Failed to enqueue image.";
break;
case SoraImageRemixErrorType.TooManyConcurrentTasks:
errorMessage = "You have too many Sora image generation tasks running. Please wait a moment.";
break;
case SoraImageRemixErrorType.SoraUsernameNotYetCreated:
errorMessage = "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
break;
case SoraImageRemixErrorType.SoraIsHavingProblems:
errorMessage = "Sora is having problems. Please try again later.";
break;
}
}
toast.error(errorMessage);
}
};
const handleWebEnqueue = async () => {
@@ -357,7 +413,7 @@ export const PromptBox2D = ({
if (!prompt.trim()) return;
try {
console.log(
"Enqueuing with prompt:",
"(4) Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages
@@ -42,19 +42,20 @@ import { PromptsApi } from "@storyteller/api";
import { SoundRegistry } from "@storyteller/soundboard";
import { toast } from "@storyteller/ui-toaster";
interface ReferenceImage {
id: string;
url: string;
file: File;
mediaToken: string;
}
import { EngineApi } from "@storyteller/api";
import { CameraSettingsModal } from "@storyteller/ui-camera-settings-modal";
import { twMerge } from "tailwind-merge";
import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal";
import { Modal } from "@storyteller/ui-modal";
import { Signal } from "@preact/signals-react";
import { CommandSuccessStatus, GetAppPreferences, SoraImageRemix, SoraImageRemixErrorType } from "@storyteller/tauri-api";
interface ReferenceImage {
id: string;
url: string;
file: File;
mediaToken: string;
}
interface PromptBox3DProps {
cameras: Signal<Camera[]>;
@@ -317,14 +318,12 @@ export const PromptBox3D = ({
const handleEnqueue = async () => {
const isDesktop = IsDesktopApp();
console.log("Is this a desktop app?", isDesktop);
const r = SoundRegistry.getInstance();
r.playSound("test");
console.log("Is Desktop?", isDesktop);
if (isDesktop) {
console.log("Desktop app - tauri enqueue");
await handleTauriEnqueue();
} else {
console.log("Web app - web enqueue");
await handleWebEnqueue();
}
};
@@ -348,7 +347,7 @@ export const PromptBox3D = ({
sceneMediaToken: "",
});
console.log("useSystemPrompt", useSystemPrompt);
console.log("(web 3d) useSystemPrompt", useSystemPrompt);
const response = await promptsApi.enqueueImageGeneration({
disableSystemPrompt: !useSystemPrompt,
@@ -369,7 +368,7 @@ export const PromptBox3D = ({
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
"(3D.2) Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages
@@ -388,6 +387,15 @@ export const PromptBox3D = ({
setisEnqueueing(true);
setTimeout(() => {
// TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if
// the backend hangs on a given request. (Sometimes Sora doesn't respond.) Ideally what we would do instead is only
// force the prompt box to be enabled again as long as another request isn't running, ie. if we just fired off two
// requests in succession, we wouldn't want to turn it off as the first one gets submitted.
console.debug('Turn off blocking of prompt box...');
setisEnqueueing(false);
}, 10000);
setPrompt(prompt);
const promptsApi = new PromptsApi();
@@ -401,31 +409,71 @@ export const PromptBox3D = ({
//sceneMediaToken: "",
});
const generateResponse = await invoke("sora_image_remix_command", {
request: {
snapshot_media_token: snapshotResult.data!,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken
),
maybe_number_of_samples: 1,
},
console.log("snapshotResult", snapshotResult);
const generateResponse = await SoraImageRemix({
snapshot_media_token: snapshotResult.data!,
disable_system_prompt: !useSystemPrompt,
prompt: prompt,
maybe_additional_images: referenceImages.map(
(image) => image.mediaToken,
),
maybe_number_of_samples: 1,
});
console.log("response", generateResponse);
console.log("generateResponse", generateResponse);
if (generateResponse.status === CommandSuccessStatus.Success) {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_success_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
toast.success("Image added to queue");
setisEnqueueing(false);
return;
} else {
const prefs = await GetAppPreferences();
const soundName = prefs.preferences?.enqueue_failure_sound;
if (soundName !== undefined) {
const registry = SoundRegistry.getInstance();
registry.playSound(soundName);
}
let errorMessage = "Failed to enqueue image";
if ("error_type" in generateResponse) {
switch (generateResponse.error_type) {
case SoraImageRemixErrorType.SoraLoginRequired:
errorMessage = "You need to log into Sora to continue. See the settings menu.";
break;
case SoraImageRemixErrorType.ServerError:
errorMessage = "Server error. Failed to enqueue image.";
break;
case SoraImageRemixErrorType.TooManyConcurrentTasks:
errorMessage = "You have too many Sora image generation tasks running. Please wait a moment.";
break;
case SoraImageRemixErrorType.SoraUsernameNotYetCreated:
errorMessage = "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
break;
case SoraImageRemixErrorType.SoraIsHavingProblems:
errorMessage = "Sora is having problems. Please try again later.";
break;
}
}
toast.error(errorMessage);
setisEnqueueing(false);
return;
}
//if (generateResponse.errorMessage) {
// handleError(response.errorMessage);
// setisEnqueueing(false);
// return;
//}
}
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
"(3D.1) Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages
@@ -443,7 +491,7 @@ export const PromptBox3D = ({
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleWebEnqueue();
handleEnqueue();
}
};
@@ -2,6 +2,9 @@
"files": [],
"include": [],
"references": [
{
"path": "../../tauri-api"
},
{
"path": "../camera-settings-modal"
},
@@ -43,6 +43,9 @@
"src/**/*.tsx"
],
"references": [
{
"path": "../../tauri-api/tsconfig.lib.json"
},
{
"path": "../camera-settings-modal/tsconfig.lib.json"
},

Some files were not shown because too many files have changed in this diff Show More