Compare commits

...
9 Commits
Author SHA1 Message Date
Tanish Manku 942005282a Some bug fixes and improvements for FK 2025-04-04 00:38:31 -07:00
Tanish Manku 14474e68c2 Add basic Forward Kinematics support (#475)
There's still some glitches around selection and deletion.
2025-04-04 00:34:09 -07:00
Michael Chung 2f44da8ac7 Changed api calls to use file upload and uuid idempotency token and stuff works. 2025-04-03 23:10:23 -04:00
Brandon Thomas 3d67f86a19 set sora creds externally 2025-04-03 22:26:38 -04:00
Brandon Thomas 2913d91d0d fix media file downloading 2025-04-03 18:47:57 -04:00
ArEnScandMichael Chung ceb77505e1 Feature/gen (#474)
Co-authored-by: Michael Chung <michaelchung@michaels-mbp-2.home>
2025-04-03 18:01:29 -04:00
ArEnScandMichael Chung 0cad3fe7da Wire screenshot upload and generate to the engine. (#470)
Co-authored-by: Michael Chung <michaelchung@michaels-mbp-2.home>
2025-04-03 17:38:34 -04:00
Tanish Manku 1ced74040f Fix: Letterboxing blocking pointer events 2025-04-03 14:09:35 -07:00
Brandon ThomasandKasisnu c9dd545ae1 Use OpenAI Sora GPT 4o image gen (#472)
Co-authored-by: Kasisnu <5044120+kasisnu@users.noreply.github.com>
2025-04-03 17:07:20 -04:00
53 changed files with 2784 additions and 100 deletions
Generated
+341 -44
View File
@@ -646,9 +646,9 @@ checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e"
[[package]]
name = "anyhow"
version = "1.0.86"
version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
dependencies = [
"backtrace",
]
@@ -680,7 +680,7 @@ dependencies = [
"ort",
"primitives",
"rand 0.8.5",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_json",
"serde_yaml",
@@ -2399,10 +2399,29 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9"
dependencies = [
"cookie 0.18.1",
"document-features",
"idna 1.0.3",
"log",
"publicsuffix",
"serde",
"serde_derive",
"serde_json",
"time",
"url",
]
[[package]]
name = "cookies"
version = "0.0.1"
@@ -3340,6 +3359,15 @@ dependencies = [
"const-random",
]
[[package]]
name = "document-features"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d"
dependencies = [
"litrs",
]
[[package]]
name = "dotenv"
version = "0.15.0"
@@ -4836,7 +4864,7 @@ dependencies = [
"errors",
"jwt-simple",
"my-workspace-hack",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_derive",
"serde_json",
@@ -5062,7 +5090,7 @@ dependencies = [
"native-tls",
"num_cpus",
"rand 0.8.5",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_json",
"thiserror 2.0.11",
@@ -5437,6 +5465,124 @@ dependencies = [
"png",
]
[[package]]
name = "icu_collections"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
dependencies = [
"displaydoc",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_locid"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
dependencies = [
"displaydoc",
"litemap",
"tinystr",
"writeable",
"zerovec",
]
[[package]]
name = "icu_locid_transform"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
dependencies = [
"displaydoc",
"icu_locid",
"icu_locid_transform_data",
"icu_provider",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_locid_transform_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
[[package]]
name = "icu_normalizer"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
dependencies = [
"displaydoc",
"icu_collections",
"icu_normalizer_data",
"icu_properties",
"icu_provider",
"smallvec 1.13.2",
"utf16_iter",
"utf8_iter",
"write16",
"zerovec",
]
[[package]]
name = "icu_normalizer_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
[[package]]
name = "icu_properties"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
dependencies = [
"displaydoc",
"icu_collections",
"icu_locid_transform",
"icu_properties_data",
"icu_provider",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_properties_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
[[package]]
name = "icu_provider"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
dependencies = [
"displaydoc",
"icu_locid",
"icu_provider_macros",
"stable_deref_trait",
"tinystr",
"writeable",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_provider_macros"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "idempotency"
version = "0.0.1"
@@ -5461,6 +5607,27 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "idna"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
dependencies = [
"idna_adapter",
"smallvec 1.13.2",
"utf8_iter",
]
[[package]]
name = "idna_adapter"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
dependencies = [
"icu_normalizer",
"icu_properties",
]
[[package]]
name = "image"
version = "0.25.6"
@@ -5646,7 +5813,7 @@ dependencies = [
"rand 0.8.5",
"redis_common",
"redis_schema",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_derive",
"serde_json",
@@ -5880,10 +6047,11 @@ checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e"
[[package]]
name = "js-sys"
version = "0.3.65"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
dependencies = [
"once_cell",
"wasm-bindgen",
]
@@ -6058,7 +6226,7 @@ dependencies = [
"futures-util",
"hostname 0.3.1",
"httpdate",
"idna",
"idna 0.5.0",
"mime",
"native-tls",
"nom",
@@ -6278,6 +6446,18 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9"
[[package]]
name = "litemap"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
[[package]]
name = "litrs"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
[[package]]
name = "local-channel"
version = "0.1.5"
@@ -6844,7 +7024,7 @@ dependencies = [
"ort",
"primitives",
"rand 0.8.5",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_json",
"serde_yaml",
@@ -7021,7 +7201,7 @@ dependencies = [
"regex",
"regex-automata",
"regex-syntax",
"reqwest 0.12.12",
"reqwest 0.12.15",
"rsa",
"rustls-pki-types",
"semver 1.0.23",
@@ -7807,6 +7987,25 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openai_sora_client"
version = "0.0.1"
dependencies = [
"anyhow",
"crockford",
"errors",
"filesys",
"log",
"my-workspace-hack",
"reqwest 0.12.15",
"serde",
"serde_derive",
"serde_json",
"subprocess_common",
"testing",
"tokio",
]
[[package]]
name = "openssl"
version = "0.10.71"
@@ -8725,6 +8924,12 @@ dependencies = [
"prost",
]
[[package]]
name = "psl-types"
version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
[[package]]
name = "psm"
version = "0.1.21"
@@ -8754,6 +8959,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "publicsuffix"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
dependencies = [
"idna 1.0.3",
"psl-types",
]
[[package]]
name = "pulldown-cmark"
version = "0.9.3"
@@ -9483,12 +9698,14 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.12.12"
version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb"
dependencies = [
"base64 0.22.1",
"bytes 1.10.0",
"cookie 0.18.1",
"cookie_store",
"encoding_rs",
"futures-channel",
"futures-core",
@@ -9505,6 +9722,7 @@ dependencies = [
"js-sys",
"log",
"mime",
"mime_guess",
"native-tls",
"once_cell",
"percent-encoding",
@@ -9717,7 +9935,7 @@ dependencies = [
"minidom",
"percent-encoding",
"quick-xml 0.36.2",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_derive",
"serde_json",
@@ -10171,9 +10389,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
version = "1.0.210"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
dependencies = [
"serde_derive",
]
@@ -10191,9 +10409,9 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.210"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
dependencies = [
"proc-macro2",
"quote",
@@ -10226,9 +10444,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.128"
version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
dependencies = [
"indexmap 2.2.6",
"itoa 1.0.11",
@@ -11076,6 +11294,7 @@ dependencies = [
"mysql_queries",
"native-tls",
"once_cell",
"openai_sora_client",
"password",
"pin-project-lite",
"primitives",
@@ -11087,7 +11306,7 @@ dependencies = [
"redis_common",
"redis_schema",
"regex",
"reqwest 0.12.12",
"reqwest 0.12.15",
"reusable_types",
"rust-s3",
"serde",
@@ -11851,7 +12070,7 @@ dependencies = [
"percent-encoding",
"plist",
"raw-window-handle",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_json",
"serde_repr",
@@ -12210,7 +12429,7 @@ dependencies = [
"easyenv",
"log",
"my-workspace-hack",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde_json",
]
@@ -12278,6 +12497,16 @@ dependencies = [
"crunchy",
]
[[package]]
name = "tinystr"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
dependencies = [
"displaydoc",
"zerovec",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
@@ -12899,7 +13128,7 @@ dependencies = [
"my-workspace-hack",
"mysql_queries",
"r2d2_redis",
"reqwest 0.12.12",
"reqwest 0.12.15",
"serde",
"serde_derive",
"serde_json",
@@ -13160,7 +13389,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
dependencies = [
"form_urlencoded",
"idna",
"idna 0.5.0",
"percent-encoding",
"serde",
]
@@ -13219,12 +13448,24 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "utf16_iter"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8-width"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
[[package]]
name = "utf8_iter"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -13265,7 +13506,7 @@ dependencies = [
"actix-web",
"mime_guess",
"regex",
"reqwest 0.12.12",
"reqwest 0.12.15",
"rust-embed",
"serde",
"serde_json",
@@ -13502,23 +13743,24 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
dependencies = [
"cfg-if 1.0.0",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.92"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.87",
@@ -13539,9 +13781,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.92"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -13549,9 +13791,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.92"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
@@ -13562,9 +13804,12 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.92"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
dependencies = [
"unicode-ident",
]
[[package]]
name = "wasm-streams"
@@ -13610,7 +13855,7 @@ dependencies = [
"log",
"my-workspace-hack",
"once_cell",
"reqwest 0.12.12",
"reqwest 0.12.15",
"rss",
"scraper",
"serde",
@@ -13864,7 +14109,7 @@ dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.2.0",
"windows-strings",
"windows-strings 0.1.0",
"windows-targets 0.52.6",
]
@@ -13898,13 +14143,13 @@ checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
[[package]]
name = "windows-registry"
version = "0.2.0"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
dependencies = [
"windows-result 0.2.0",
"windows-strings",
"windows-targets 0.52.6",
"windows-result 0.3.2",
"windows-strings 0.3.1",
"windows-targets 0.53.0",
]
[[package]]
@@ -13925,6 +14170,15 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
@@ -13935,6 +14189,15 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
@@ -14269,6 +14532,18 @@ dependencies = [
"bitflags 2.5.0",
]
[[package]]
name = "write16"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
[[package]]
name = "writeable"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
[[package]]
name = "wry"
version = "0.48.1"
@@ -14448,6 +14723,28 @@ dependencies = [
"syn 2.0.87",
]
[[package]]
name = "zerovec"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
dependencies = [
"yoke",
"zerofrom",
"zerovec-derive",
]
[[package]]
name = "zerovec-derive"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "zip"
version = "0.6.6"
+21
View File
@@ -32,6 +32,7 @@ members = [
"crates/lib/api_clients/cloud_storage",
"crates/lib/api_clients/google_drive_common",
"crates/lib/api_clients/google_sign_in",
"crates/lib/api_clients/openai_sora_client",
"crates/lib/api_clients/thumbnail_generator",
"crates/lib/bootstrap",
"crates/lib/bucket_paths",
@@ -118,3 +119,23 @@ members = [
# see usage: https://crates.io/crates/cargo-hakari
"crates/my-workspace-hack",
]
[workspace.package]
publish = false
[workspace.dependencies]
# Hakari workspace hack (faster compile times)
my-workspace-hack = { path = "crates/my-workspace-hack" }
# Internal libs
errors = { path = "crates/lib/errors" }
# External libs
# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
anyhow = "1.0.97"
log = "0.4.27"
reqwest = { version = "0.12.15", features = ["json", "cookies"] }
serde = "1.0.219"
serde_derive = "1.0.219"
serde_json = "1.0.140"
@@ -40,6 +40,10 @@ pub enum InferenceJobType {
#[serde(rename = "studio_gen2")]
StudioGen2,
/// Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
/// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
#[serde(rename = "convert_fbx_gltf")]
ConvertFbxToGltf,
@@ -108,6 +112,7 @@ impl InferenceJobType {
Self::GptSovits => "gpt_sovits",
Self::ComfyUi => "comfy_ui",
Self::StudioGen2 => "studio_gen2",
Self::ImageGenApi => "image_gen_api",
Self::ConvertFbxToGltf => "convert_fbx_gltf",
Self::MocapNet => "mocap_net",
Self::RvcV2 => "rvc_v2",
@@ -132,6 +137,7 @@ impl InferenceJobType {
"gpt_sovits" => Ok(Self::GptSovits),
"comfy_ui" => Ok(Self::ComfyUi),
"studio_gen2" => Ok(Self::StudioGen2),
"image_gen_api" => Ok(Self::ImageGenApi),
"convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
"mocap_net" => Ok(Self::MocapNet),
"rvc_v2" => Ok(Self::RvcV2),
@@ -159,6 +165,7 @@ impl InferenceJobType {
Self::GptSovits,
Self::ComfyUi,
Self::StudioGen2,
Self::ImageGenApi,
Self::ConvertFbxToGltf,
Self::MocapNet,
Self::RvcV2,
@@ -197,6 +204,7 @@ mod tests {
assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
assert_serialization(InferenceJobType::StudioGen2, "studio_gen2");
assert_serialization(InferenceJobType::ImageGenApi, "image_gen_api");
assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
assert_serialization(InferenceJobType::MocapNet, "mocap_net");
assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
@@ -220,6 +228,7 @@ mod tests {
assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
assert_eq!(InferenceJobType::StudioGen2.to_str(), "studio_gen2");
assert_eq!(InferenceJobType::ImageGenApi.to_str(), "image_gen_api");
assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
@@ -243,6 +252,7 @@ mod tests {
assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
assert_eq!(InferenceJobType::from_str("studio_gen2").unwrap(), InferenceJobType::StudioGen2);
assert_eq!(InferenceJobType::from_str("image_gen_api").unwrap(), InferenceJobType::ImageGenApi);
assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
@@ -261,7 +271,7 @@ mod tests {
fn all_variants() {
// Static check
let mut variants = InferenceJobType::all_variants();
assert_eq!(variants.len(), 16);
assert_eq!(variants.len(), 17);
assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
@@ -269,6 +279,7 @@ mod tests {
assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
@@ -33,6 +33,10 @@ pub enum InferenceModelType {
#[serde(rename = "seed_vc")]
SeedVc,
/// NB: This is for Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
#[serde(rename = "tacotron2")]
Tacotron2,
#[serde(rename = "vits")]
@@ -70,6 +74,7 @@ impl InferenceModelType {
Self::VallEX => "vall_e_x",
Self::RerenderAVideo => "rerender_a_video",
Self::StableDiffusion => "stable_diffusion",
Self::ImageGenApi => "image_gen_api",
Self::SeedVc => "seed_vc",
Self::MocapNet => "mocap_net",
Self::StyleTTS2 => "styletts2",
@@ -90,6 +95,7 @@ impl InferenceModelType {
"vall_e_x" => Ok(Self::VallEX),
"rerender_a_video" => Ok(Self::RerenderAVideo),
"stable_diffusion" => Ok(Self::StableDiffusion),
"image_gen_api" => Ok(Self::ImageGenApi),
"mocap_net" => Ok(Self::MocapNet),
"styletts2" => Ok(Self::StyleTTS2),
"comfy_ui" => Ok(Self::ComfyUi),
@@ -113,6 +119,7 @@ impl InferenceModelType {
Self::VallEX,
Self::RerenderAVideo,
Self::StableDiffusion,
Self::ImageGenApi,
Self::MocapNet,
Self::StyleTTS2,
Self::ConvertFbxToGltf,
@@ -140,6 +147,7 @@ mod tests {
assert_serialization(InferenceModelType::VallEX, "vall_e_x");
assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video");
assert_serialization(InferenceModelType::StableDiffusion, "stable_diffusion");
assert_serialization(InferenceModelType::ImageGenApi, "image_gen_api");
assert_serialization(InferenceModelType::MocapNet, "mocap_net");
assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
@@ -159,6 +167,7 @@ mod tests {
assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x");
assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video");
assert_eq!(InferenceModelType::StableDiffusion.to_str(), "stable_diffusion");
assert_eq!(InferenceModelType::ImageGenApi.to_str(), "image_gen_api");
assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
@@ -176,6 +185,7 @@ mod tests {
assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX);
assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo);
assert_eq!(InferenceModelType::from_str("stable_diffusion").unwrap(), InferenceModelType::StableDiffusion);
assert_eq!(InferenceModelType::from_str("image_gen_api").unwrap(), InferenceModelType::ImageGenApi);
assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
@@ -187,17 +197,18 @@ mod tests {
fn all_variants() {
// Static check
let mut variants = InferenceModelType::all_variants();
assert_eq!(variants.len(), 13);
assert_eq!(variants.len(), 15);
assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));
assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX));
assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StableDiffusion));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
@@ -91,6 +91,10 @@ pub enum PublicInferenceJobType {
/// Re-render a video is a video style transfer algorithm. We developed code
/// around it, but chose to develop AnimateDiff / ComfyUI support instead.
RerenderAVideo,
/// Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
}
/// NB: Legacy API for older code.
@@ -119,6 +123,7 @@ impl PublicInferenceJobType {
InferenceJobType::Unknown => Self::Unknown,
InferenceJobType::BevyToWorkflow => Self::BevyToWorkflow,
InferenceJobType::RerenderAVideo => Self::RerenderAVideo,
InferenceJobType::ImageGenApi => Self::ImageGenApi,
}
}
@@ -140,6 +145,7 @@ impl PublicInferenceJobType {
Self::SeedVc => InferenceJobType::SeedVc,
Self::SoVitsSvc => InferenceJobType::SoVitsSvc,
Self::StableDiffusion => InferenceJobType::StableDiffusion,
Self::ImageGenApi => InferenceJobType::ImageGenApi,
Self::StyleTTS2 => InferenceJobType::StyleTTS2,
Self::Tacotron2 => InferenceJobType::Tacotron2,
Self::Unknown => InferenceJobType::Unknown,
@@ -234,17 +234,31 @@ impl BucketClient {
.map(|s| s.to_string())
.ok_or(anyhow!("could not convert object path to string"))?;
info!("downloading from bucket: {:?}", &object_path_str);
info!("creating file for bucket download: {:?}", filesystem_path.as_ref());
let mut output_file = tokio::fs::File::create(filesystem_path).await?;
let mut output_file = File::create(filesystem_path).await?;
let status_code = self.bucket.get_object_to_writer(&object_path_str, &mut output_file).await?;
let result = self.bucket.get_object_to_writer(&object_path_str, &mut output_file).await;
info!("downloading from bucket (named '{}'), path: {}", &self.bucket.name, &object_path_str);
let status_code = match result {
Ok(status_code) => status_code,
Err(err) => {
return bail!("Error downloading from bucket (named '{}'): {:?}", &self.bucket.name, err)
}
};
match status_code {
404 => bail!("File not found in bucket: {}", &object_path_str),
_ => {},
404 => {
error!("File not found in bucket (named '{}'), path: {}", &self.bucket.name, &object_path_str);
bail!("File not found in bucket: {}", &object_path_str)
},
_ => {
info!("download code: {}", status_code);
},
}
info!("download code: {}", status_code);
Ok(())
}
}
@@ -0,0 +1,35 @@
[package]
name = "openai_sora_client"
edition = "2021"
version = "0.0.1"
publish = false
[package.metadata.cargo-udeps.ignore]
normal = ["my-workspace-hack"]
[lib]
name = "openai_sora_client"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
my-workspace-hack = { workspace = true }
# Internal
crockford = { path = "../../data/crockford" }
errors = { path = "../../errors" }
subprocess_common = { path = "../../subprocess_common" }
# External
anyhow = { workspace = true }
log = { workspace = true }
reqwest = { workspace = true, features = ["multipart"] }
serde = { workspace = true }
serde_derive = { workspace = true }
serde_json = { workspace = true }
tokio = { version = "1.38.0", features = ["macros"] }
[dev-dependencies]
testing = { path = "../../testing" }
filesys = { path = "../../files/filesys" }
@@ -0,0 +1,43 @@
use reqwest::RequestBuilder;
use errors::AnyhowResult;
// NB: It appears that the sentinel may require a matching user agent.
pub (crate) const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
#[derive(Clone)]
pub struct SoraCredentials {
pub bearer_token: String,
pub cookie: String,
pub sentinel: Option<String>,
}
impl SoraCredentials {
pub fn authorization_header_value(&self) -> String {
match self.bearer_token.get(0..6) {
Some("bearer") | Some("Bearer") => self.bearer_token.clone(),
_ => "Bearer ".to_owned() + &self.bearer_token,
}
}
pub fn from_env() -> AnyhowResult<Self> {
let bearer = std::env::var("SORA_BEARER_TOKEN")?;
let cookie = std::env::var("SORA_COOKIE")?;
let sentinel = std::env::var("SORA_SENTINEL").ok();
Ok(Self { bearer_token: bearer, cookie, sentinel })
}
pub fn add_credential_headers_to_request(&self, request: RequestBuilder) -> RequestBuilder {
let bearer_header = self.authorization_header_value();
let mut request = request
.header("User-Agent", USER_AGENT)
.header("Cookie", &self.cookie)
.header("Authorization", bearer_header);
if let Some(sentinel) = &self.sentinel {
request = request.header("OpenAI-Sentinel-Token", sentinel);
}
request
}
}
@@ -0,0 +1,48 @@
#[derive(Debug, Clone)]
pub struct SoraImageGenResponse {
pub task_id: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NumImages {
One,
Two,
Four
}
impl NumImages {
pub fn as_count(&self) -> usize {
match self {
NumImages::One => 1,
NumImages::Two => 2,
NumImages::Four => 4,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ImageSize {
Square,
Wide,
Tall,
}
impl ImageSize {
// TODO: Verify these dimensions. 480x480 is correct.
pub fn as_dimensions(&self) -> (u16, u16) {
match self {
ImageSize::Square => (480, 480),
ImageSize::Wide => (854, 480),
ImageSize::Tall => (480, 854),
}
}
pub fn as_width(&self) -> u16 {
self.as_dimensions().0
}
pub fn as_height(&self) -> u16 {
self.as_dimensions().1
}
}
@@ -0,0 +1,130 @@
use anyhow::anyhow;
use serde_derive::{Deserialize, Serialize};
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
/// This user agent is tied to the sentinel generation. If we need to change it, we may need to change sentinel generation too.
const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) enum VideoGenType {
ImageGen,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) enum OperationType {
/// Simple prompt without reference images
SimpleCompose,
/// Prompt with reference images
Remix,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) enum InpaintItemType {
Image,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct InpaintItem {
pub r#type: InpaintItemType,
/// Typically "0"
pub frame_index: u32,
/// Unknown field; defaults to null
pub preset_id: Option<String>,
/// Unknown field; defaults to null
pub generation_id: Option<String>,
/// Token identifier of the item, eg. "media_01jqt9vt20erx9zvryf3v1pecx"
pub upload_media_id: String,
/// Typically "0"
pub source_start_frame: u32,
/// Typically "0"
pub source_end_frame: u32,
/// Unknown field; defaults to null
pub crop_bounds: Option<String>,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct RawSoraImageGenRequest {
/// eg. "image_gen"
pub r#type: VideoGenType,
/// eg. "simple_compose"
pub operation: OperationType,
/// The user's raw prompt
pub prompt: String,
/// The number of variants to generate, eg. 2
pub n_variants: usize,
/// The width of the image, eg. 480
pub width: u16,
/// The height of the image, eg. 480
pub height: u16,
/// The number of frames to generate, eg. 1
pub n_frames: u8,
/// The items to inpaint, eg. []
pub inpaint_items: Vec<InpaintItem>,
}
#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct RawSoraResponse {
/// eg. "task_01jqsz9dsae9tvjygf1abrv3xf"
pub id: String,
/// not known
pub priority: Option<String>,
}
// https://sora.com/backend/notif?limit=100&before=task_01jqpg8qghenet0rw2a79p0vbn
// error
// message "Missing bearer authentication in header"
// type "invalid_request_error"
// param null
// code null
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct RawSoraErrorResponse {
error: String,
message: String,
r#type: String,
param: Option<String>,
code: Option<String>,
}
/// Don't expose the internal request implementation as there are only a few "correct" ways to call the API.
pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: &SoraCredentials) -> AnyhowResult<RawSoraResponse> {
let client = reqwest::Client::new();
let sentinel = match credentials.sentinel.as_deref() {
None => return Err(anyhow!("Calls to image gen require a sentinel in the credentials payload.")),
Some(sentinel) => sentinel,
};
let http_request = client.post(SORA_IMAGE_GEN_URL)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie)
.header("Authorization", credentials.authorization_header_value())
.header("Content-Type", "application/json")
.header("OpenAI-Sentinel-Token", sentinel);
let http_request = http_request.json(&sora_request).build()?;
let response = client.execute(http_request)
.await?;
let status = response.status();
let response_body = &response.text().await?;
if status != reqwest::StatusCode::OK {
return Err(anyhow::anyhow!("the request failed; status = {:?}, message = {:?}", status, response_body));
}
let response = serde_json::from_str(response_body)?;
Ok(response)
}
@@ -0,0 +1,4 @@
pub (crate) mod image_gen_http_request;
pub mod common;
pub mod sora_image_gen_remix;
pub mod sora_image_gen_simple;
@@ -0,0 +1,87 @@
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, VideoGenType};
pub struct SoraImageGenRemixRequest<'a> {
pub prompt: String,
pub num_images: NumImages,
pub image_size: ImageSize,
pub sora_media_tokens: Vec<String>,
pub credentials: &'a SoraCredentials,
}
/// The "remix" commands let you supply additional images as context.
/// Sora "media tokens" of previously uploaded images must be supplied.
pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
let args = RawSoraImageGenRequest {
r#type: VideoGenType::ImageGen,
operation: OperationType::Remix,
prompt: request.prompt,
n_variants: request.num_images.as_count(),
width: request.image_size.as_width(),
height: request.image_size.as_height(),
n_frames: 1,
inpaint_items: request.sora_media_tokens.into_iter().map(|token| {
InpaintItem {
r#type: InpaintItemType::Image,
frame_index: 0,
preset_id: None,
generation_id: None,
upload_media_id: token,
source_start_frame: 0,
source_end_frame: 0,
crop_bounds: None,
}
}).collect(),
};
// TODO: Error handling.
let result = image_gen_http_request(args, &request.credentials).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
})
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages};
use crate::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let sentinel = read_to_string(test_file_path("test_data/temp/sentinel.txt")?)?;
let sentinel = sentinel.trim().to_string();
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
let response = sora_image_gen_remix(SoraImageGenRemixRequest {
prompt: "Match the pose and scene layout from the uploaded image exactly. A smart dog getting kisses from another dog. Brick building. Anime style".to_string(),
num_images: NumImages::One,
image_size: ImageSize::Square,
sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()],
credentials: &creds,
}).await?;
println!("task_id: {}", response.task_id);
assert!(response.task_id.starts_with("task_"));
Ok(())
}
}
@@ -0,0 +1,72 @@
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType};
pub struct SoraImageGenSimpleRequest<'a> {
pub prompt: String,
pub num_images: NumImages,
pub image_size: ImageSize,
pub credentials: &'a SoraCredentials,
}
pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
let sora_request = RawSoraImageGenRequest {
r#type: VideoGenType::ImageGen,
operation: OperationType::SimpleCompose,
prompt: args.prompt,
n_variants: args.num_images.as_count(),
width: args.image_size.as_width(),
height: args.image_size.as_height(),
n_frames: 1,
inpaint_items: vec![],
};
// TODO: Error handling.
let result = image_gen_http_request(sora_request, args.credentials).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
})
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages};
use crate::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest};
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let sentinel = read_to_string(test_file_path("test_data/temp/sentinel.txt")?)?;
let sentinel = sentinel.trim().to_string();
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
let response = sora_image_gen_simple(SoraImageGenSimpleRequest {
prompt: "A pirate and a ninja fight in a battle inside a UFO. Fully photo realistic, lifelike, lens flare".to_string(),
num_images: NumImages::One,
image_size: ImageSize::Square,
credentials: &creds,
}).await?;
println!("task_id: {}", response.task_id);
assert!(response.task_id.starts_with("task_"));
Ok(())
}
}
@@ -0,0 +1 @@
pub mod sora_job_status;
@@ -0,0 +1,64 @@
const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
// https://sora.com/backend/notif?limit=100&before=task_01jqwwrkvgeqp8jsf5mqk1jceh
use crate::credentials::SoraCredentials;
pub async fn sora_job_status(
task_id: &str,
credentials: &SoraCredentials,
) -> anyhow::Result<()> {
let url = format!("https://sora.com/backend/notif?limit=100&before={}", task_id);
let client = reqwest::Client::new();
let request = client
.get(url)
.header("Authorization", &credentials.bearer_token)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie);
let response = request.send().await?;
let json_response = &response.text().await?;
println!(" >>> response = {:?}", json_response);
//let response = serde_json::from_str(json_response)?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::job::sora_job_status::sora_job_status;
#[ignore]
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let sentinel = read_to_string(test_file_path("test_data/temp/sentinel.txt")?)?;
let sentinel = sentinel.trim().to_string();
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
let task_id = "task_01jqwwrkvgeqp8jsf5mqk1jceh";
let result = sora_job_status(task_id, &creds).await?;
Ok(())
}
}
@@ -0,0 +1,4 @@
pub mod credentials;
pub mod image_gen;
pub mod job;
pub mod upload;
@@ -0,0 +1,3 @@
pub (crate) mod upload_media_http_request;
pub mod upload_media_from_bytes;
pub mod upload_media_from_file;
@@ -0,0 +1,29 @@
use std::path::{Path, PathBuf};
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
pub struct SoraMediaUploadRequest<'a> {
pub file_path: String,
pub credentials: &'a SoraCredentials,
}
/// Upload bytes.
/// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference.
pub async fn sora_media_upload_from_bytes<P: AsRef<Path>>(bytes: Vec<u8>, file_name: String, creds: &SoraCredentials) -> AnyhowResult<SoraMediaUploadResponse> {
let file_path = PathBuf::from(&file_name);
// TODO: Read file magic bytes first, then fall back to this.
let mime_type = match file_path.extension().and_then(|e| e.to_str()) {
Some("jpg") | Some("jpeg") => "image/jpeg",
Some("png") => "image/png",
// Some("webp") => "image/webp",
// Some("gif") => "image/gif",
// Some("mp4") => "video/mp4",
// Some("mov") => "video/quicktime",
// Some("webm") => "video/webm",
_ => "application/octet-stream",
};
Ok(upload_media_http_request(bytes, file_name, mime_type, creds).await?)
}
@@ -0,0 +1,81 @@
use std::path::Path;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
/// Try to prevent buffer reallocations.
/// There's a better way to implement this.
const INITIAL_BUFFER_SIZE : usize = 1024*1024;
pub struct SoraMediaUploadRequest<'a> {
pub file_path: String,
pub credentials: &'a SoraCredentials,
}
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: &SoraCredentials) -> AnyhowResult<SoraMediaUploadResponse> {
let mut file = File::open(&file_path).await?;
let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE);
file.read_to_end(&mut buffer).await?;
let filename = file_path.as_ref().file_name()
.ok_or_else(|| anyhow::anyhow!("Invalid file path"))?
.to_string_lossy()
.to_string();
// TODO: Read file magic bytes first, then fall back to this.
let mime_type = match file_path.as_ref().extension().and_then(|e| e.to_str()) {
Some("jpg") | Some("jpeg") => "image/jpeg",
Some("png") => "image/png",
// Some("webp") => "image/webp",
// Some("gif") => "image/gif",
// Some("mp4") => "video/mp4",
// Some("mov") => "video/quicktime",
// Some("webm") => "video/webm",
_ => "application/octet-stream",
};
Ok(upload_media_http_request(buffer, filename, mime_type, creds).await?)
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::upload::upload_media_from_file::sora_media_upload_from_file;
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let image_path = test_file_path("test_data/image/juno.jpg")?; // media_01jqyqgqpwf40tkcapq5bmaz5d
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: None,
};
let response = sora_media_upload_from_file(
image_path,
&creds,
).await?;
println!("media: {:?}", response);
println!("media_id: {}", response.id);
println!("media_url: {}", response.url);
assert!(response.id.starts_with("media_"));
Ok(())
}
}
@@ -0,0 +1,81 @@
use crate::credentials::{SoraCredentials, USER_AGENT};
use errors::AnyhowResult;
use reqwest::multipart::{Form, Part};
use reqwest::Client;
use serde::Deserialize;
const SORA_UPLOAD_MEDIA_URL: &str = "https://sora.com/backend/uploads";
// Content-Disposition: form-data; name="file"; filename="imagename.webp"
/*
Response:
{
"id": "media_01jqt9vt20erx9zvryf3v1pecx",
"type": "image",
"created_at": "2025-04-02T03:49:04.185867Z",
"filename": "Rith_essa.webp",
"extension": "jpeg",
"mime_type": "image/jpeg",
"url": "https://videos.openai.com/vg-assets/assets%2Fclient_upload%2Fmedia%2Fuser-9wf6JFdgRSJLjvSJ53LNAbV8%2Fmedia_01jqt9vt20erx9zvryf3v1pecx.jpeg?st=2025-04-02T02%3A39%3A01Z&se=2025-04-08T03%3A39%3A01Z&sks=b&skt=2025-04-02T02%3A39%3A01Z&ske=2025-04-08T03%3A39%3A01Z&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skoid=3d249c53-07fa-4ba4-9b65-0bf8eb4ea46a&skv=2019-02-02&sv=2018-11-09&sr=b&sp=r&spr=https%2Chttp&sig=%2BdnTQyw9OFF2kaVoEReT2cVfTqmKjGt18m7P4wKCWCI%3D&az=oaivgprodscus",
"width": 1200,
"height": 675,
"duration_sec": null,
"n_frames": 1,
"size_bytes": 118501,
"thumbnail_url": "https://videos.openai.com/vg-assets/assets%2Fclient_upload%2Fmedia%2Fuser-9wf6JFdgRSJLjvSJ53LNAbV8%2Fmedia_01jqt9vt20erx9zvryf3v1pecx.jpg?st=2025-04-02T02%3A39%3A01Z&se=2025-04-08T03%3A39%3A01Z&sks=b&skt=2025-04-02T02%3A39%3A01Z&ske=2025-04-08T03%3A39%3A01Z&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skoid=3d249c53-07fa-4ba4-9b65-0bf8eb4ea46a&skv=2019-02-02&sv=2018-11-09&sr=b&sp=r&spr=https%2Chttp&sig=9aU9xKI0BkhRtrTb0RD05%2BT6prBnmXk1I7nV2nTsFbY%3D&az=oaivgprodscus"
}
*/
#[derive(Debug, Deserialize)]
pub struct SoraMediaUploadResponse {
/// This is the media token id.
pub id: String,
pub r#type: String,
pub created_at: String,
pub filename: String,
pub extension: String,
pub mime_type: String,
pub url: String,
pub width: Option<u32>,
pub height: Option<u32>,
pub duration_sec: Option<f64>,
pub n_frames: Option<u32>,
pub size_bytes: u64,
pub thumbnail_url: Option<String>,
}
pub (crate) struct SoraMediaUploadRequest<'a> {
pub file_path: String,
pub credentials: &'a SoraCredentials,
}
pub (crate) async fn upload_media_http_request(file_bytes: Vec<u8>, filename: String, mime_type: &str, credentials: &SoraCredentials) -> AnyhowResult<SoraMediaUploadResponse> {
// Create multipart form
let part = Part::bytes(file_bytes) // NB: Reqwest needs to own the bytes.
.file_name(filename) // NB: Reqwest needs to own the bytes
.mime_str(mime_type)?;
let form = Form::new().part("file", part);
// Make API request
let client = Client::new();
let request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
.multipart(form)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie)
.header("Authorization", credentials.authorization_header_value());
let response = request_builder.send().await?;
// Check response status
if !response.status().is_success() {
let status = response.status();
let text = response.text().await?;
return Err(anyhow::anyhow!("Upload failed with status {}: {}", status, text));
}
// Parse response
let upload_response = response.json::<SoraMediaUploadResponse>().await?;
Ok(upload_response)
}
@@ -1,7 +1,6 @@
use crate::payloads::generic_inference_args::inner_payloads::f5_tts_payload::F5TTSPayload;
use crate::payloads::generic_inference_args::inner_payloads::face_fusion_payload::FaceFusionPayload;
use crate::payloads::generic_inference_args::inner_payloads::gptsovits_payload::GptSovitsPayload;
use crate::payloads::generic_inference_args::inner_payloads::image_generation_payload::StableDiffusionArgs;
use crate::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncArgs;
use crate::payloads::generic_inference_args::inner_payloads::live_portrait_payload::LivePortraitPayload;
use crate::payloads::generic_inference_args::inner_payloads::mocap_payload::MocapArgs;
@@ -12,6 +11,8 @@ use crate::payloads::generic_inference_args::inner_payloads::tts_payload::TTSArg
use crate::payloads::generic_inference_args::inner_payloads::videofilter_payload::RerenderArgs;
use crate::payloads::generic_inference_args::inner_payloads::workflow_payload::WorkflowArgs;
use errors::AnyhowResult;
use crate::payloads::generic_inference_args::inner_payloads::image_generation_payload::StableDiffusionArgs;
use crate::payloads::generic_inference_args::inner_payloads::sora_image_gen_args::SoraImageGenArgs;
/// Used to encode extra state for the `generic_inference_jobs` table.
/// This should act somewhat like a serialized protobuf stored inside a record.
@@ -111,7 +112,7 @@ pub enum PolymorphicInferenceArgs {
Tts(TTSArgs),
// No arguments yet.
// It might be best to just not include this when not used.
/// Voice conversion. (Short name to save space when serializing.)
Vc {
/// Argument for so-vits-svc
@@ -143,6 +144,10 @@ pub enum PolymorphicInferenceArgs {
/// Image generation. (Short name to save space when serializing.)
Ig(StableDiffusionArgs),
/// Sora Image Generation. (Short name to save space when serializing.)
Sg(SoraImageGenArgs),
/// Mocap (Short name to save space when serializing.)
Mc(MocapArgs),
@@ -1,3 +1,4 @@
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
/// Video sources can be one of several:
@@ -52,7 +53,7 @@ pub struct StableDiffusionArgs {
#[serde(rename = "se")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_seed: Option<i64>,
#[serde(rename = "mu")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_upload_path: Option<String>,
@@ -60,7 +61,7 @@ pub struct StableDiffusionArgs {
#[serde(rename = "cf")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_cfg_scale: Option<u32>,
#[serde(rename = "lu")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_lora_upload_path: Option<String>,
@@ -1,3 +1,4 @@
pub mod f5_tts_payload;
pub mod face_fusion_payload;
pub mod gptsovits_payload;
pub mod image_generation_payload;
@@ -5,9 +6,9 @@ pub mod lipsync_payload;
pub mod live_portrait_payload;
pub mod mocap_payload;
pub mod render_engine_scene_to_video_payload;
pub mod seed_vc_payload;
pub mod sora_image_gen_args;
pub mod studio_gen2_payload;
pub mod tts_payload;
pub mod videofilter_payload;
pub mod workflow_payload;
pub mod f5_tts_payload;
pub mod seed_vc_payload;
@@ -0,0 +1,23 @@
use tokens::tokens::media_files::MediaFileToken;
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct SoraImageGenArgs {
#[serde(rename = "p")]
#[serde(skip_serializing_if = "Option::is_none")]
pub prompt: Option<String>,
#[serde(rename = "sm")]
pub scene_snapshot_media_token: Option<MediaFileToken>,
#[serde(rename = "mf")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_additional_media_file_tokens: Option<Vec<MediaFileToken>>,
#[serde(rename = "sa")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_number_of_samples: Option<u32>,
#[serde(rename = "su")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_sora_media_upload_tokens: Option<Vec<String>>,
}
+2
View File
@@ -17,7 +17,9 @@ pub fn test_file_path<P: AsRef<Path>>(path_from_repo_root: P) -> AnyhowResult<Pa
.to_str()
.ok_or(anyhow!("path cannot convert to string"))?;
path.push(format!("../../../{}", file));
println!(" >>> path = {:?}", path);
Ok(path.canonicalize()?)
}
#[cfg(test)]
@@ -69,6 +69,7 @@ tts_common = { path = "../../../lib/tts_common" }
url_config = { path = "../../../lib/url_config" }
user_input_common = { path = "../../../lib/user_input_common" }
videos = { path = "../../../lib/files/videos" }
openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" }
# Internal Components
billing_component = { path = "../../plugins/billing" }
@@ -124,6 +124,8 @@ use crate::http_server::endpoints::stats::get_unified_queue_stats_handler::*;
use crate::http_server::endpoints::tags::list_tags_for_entity_handler::*;
use crate::http_server::endpoints::tags::set_tags_for_entity_handler::*;
use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::*;
use crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::*;
use crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::*;
use crate::http_server::endpoints::user_bookmarks::batch_get_user_bookmarks_handler::*;
use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::*;
use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::*;
@@ -207,6 +209,8 @@ use crate::http_server::web_utils::response_success_helpers::*;
crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_image_media_file_handler::upload_image_media_file_handler,
crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::enqueue_studio_image_generation_handler,
crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::upload_snapshot_media_file_handler,
crate::http_server::endpoints::studio_gen2::enqueue_studio_gen2_handler::enqueue_studio_gen2_handler,
crate::http_server::endpoints::media_files::upload::upload_new_engine_asset_media_file_handler::upload_new_engine_asset_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::upload_new_scene_media_file_handler,
@@ -336,6 +340,9 @@ use crate::http_server::web_utils::response_success_helpers::*;
AppStateServerInfo,
AppStateStatusAlertCategory,
AppStateStatusAlertInfo,
EnqueueStudioImageGenRequest,
EnqueueImageGenRequestSuccessResponse,
EnqueueImageGenRequestError,
AppStateSubscriptionProductKey,
AppStateUserInfo,
AppStateUserLocale,
@@ -347,6 +354,8 @@ use crate::http_server::web_utils::response_success_helpers::*;
BatchGetMediaFilesQueryParams,
BatchGetMediaFilesSuccessResponse,
BatchGetUserBookmarksError,
UploadSnapshotMediaFileForm,
UploadSnapshotMediaFileSuccessResponse,
BatchGetUserBookmarksQueryParams,
BatchGetUserBookmarksResponse,
BatchGetUserRatingError,
@@ -0,0 +1 @@
pub mod set_sora_secret_handler;
@@ -0,0 +1,137 @@
use std::fmt;
use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web, Either};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Form, Json, Path};
use anyhow::anyhow;
use log::{error, log, warn};
use r2d2_redis::redis::Commands;
use utoipa::ToSchema;
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use mysql_queries::queries::media_files::edit::update_media_file_animation_type::update_media_file_animation_type;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use tokens::tokens::media_files::MediaFileToken;
use crate::state::server_state::ServerState;
use crate::util::redis_sora_secrets::get_sora_credentials;
use crate::util::redis_sora_secrets::set_sora_credential_field;
use crate::util::redis_sora_secrets::RedisSoraCredentialSubkey;
#[derive(Deserialize, Clone, Copy)]
#[serde(rename_all = "snake_case")]
pub enum SetSoraSecretType {
Bearer,
Cookie,
Sentinel
}
#[derive(Deserialize, ToSchema)]
pub struct SetSoraSecretRequest {
pub key: SetSoraSecretType,
pub value: String,
}
#[derive(Serialize, ToSchema)]
pub struct SetSoraSecretSuccessResponse {
pub success: bool,
pub bearer: String,
pub cookie: String,
pub sentinel: String,
}
// =============== Error Response ===============
#[derive(Debug, Serialize, ToSchema)]
pub enum SetSoraSecretError {
BadInput(String),
NotAuthorized,
ServerError,
}
impl ResponseError for SetSoraSecretError {
fn status_code(&self) -> StatusCode {
match *self {
SetSoraSecretError::BadInput(_) => StatusCode::BAD_REQUEST,
SetSoraSecretError::NotAuthorized => StatusCode::UNAUTHORIZED,
SetSoraSecretError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for SetSoraSecretError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// =============== Handler ===============
/// Set a Sora secret
#[utoipa::path(
post,
tag = "Control Plane",
path = "/v1/control_plane/sora_secret",
responses(
(status = 200, description = "Success", body = SetSoraSecretSuccessResponse),
(status = 400, description = "Bad input", body = SetSoraSecretError),
(status = 401, description = "Not authorized", body = SetSoraSecretError),
(status = 500, description = "Server error", body = SetSoraSecretError),
),
params(
("request" = SetSoraSecretRequest, description = "Payload for Request"),
("path" = MediaFileTokenPathInfo, description = "Path for Request")
)
)]
pub async fn set_sora_secret_handler(
http_request: HttpRequest,
request: Form<SetSoraSecretRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<SetSoraSecretSuccessResponse>, SetSoraSecretError> {
// TODO(bt,2025-04-03): Some form of authentication should be required for this endpoint.
let value = request.value.trim().to_string();
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
set_sora_credential_field(
&mut redis,
match request.key {
SetSoraSecretType::Bearer => RedisSoraCredentialSubkey::Bearer,
SetSoraSecretType::Cookie => RedisSoraCredentialSubkey::Cookie,
SetSoraSecretType::Sentinel => RedisSoraCredentialSubkey::Sentinel,
},
&value
).map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
let credentials = get_sora_credentials(&mut redis)
.map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
// TODO(bt,2025-04-03): Probably not smart to return, but useful for debugging.
Ok(Json(SetSoraSecretSuccessResponse {
success: true,
bearer: credentials.bearer_token,
cookie: credentials.cookie,
sentinel: credentials.sentinel.unwrap_or_else(|| "".to_string()),
}))
}
@@ -507,6 +507,9 @@ fn validate_request(
Ok(())
}
fn inference_mode_from_http_request(http_request: &HttpRequest) -> Option<TypeOfInference> {
inference_mode_from_url_path(http_request.path())
}
@@ -0,0 +1,68 @@
use actix_web::HttpRequest;
use anyhow::anyhow;
use log::{error, info};
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::RedisConnectionManager;
use errors::AnyhowResult;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use openai_sora_client::credentials::SoraCredentials;
use crate::util::redis_sora_secrets::get_sora_credentials;
const SORA_BEARER_HEADER_NAME : &str = "sora-bearer";
const SORA_COOKIE_HEADER_NAME : &str = "sora-cookie";
const SORA_SENTINEL_HEADER_NAME : &str = "sora-sentinel";
pub fn get_sora_credentials_from_request(
http_request: &HttpRequest,
redis: &mut PooledConnection<RedisConnectionManager>,
) -> AnyhowResult<SoraCredentials> {
let sora_bearer = get_request_header_optional(&http_request, SORA_BEARER_HEADER_NAME);
let sora_cookie = get_request_header_optional(&http_request, SORA_COOKIE_HEADER_NAME);
let sora_sentinel = get_request_header_optional(&http_request, SORA_SENTINEL_HEADER_NAME);
match (&sora_bearer, &sora_cookie, &sora_sentinel) {
(Some(bearer), Some(cookie), Some(sentinel)) => {
info!("Sora credentials were present in HTTP request headers!");
return Ok(SoraCredentials {
bearer_token: bearer.to_string(),
cookie: cookie.to_string(),
sentinel: Some(sentinel.to_string()),
});
}
_ => {} // Fall through
}
let redis_result = get_sora_credentials(redis);
match redis_result {
Ok(mut credentials) => {
if let Some(bearer) = sora_bearer {
info!("override bearer value");
credentials.bearer_token = bearer;
}
if let Some(cookie) = sora_cookie {
info!("override cookie value");
credentials.cookie = cookie;
}
if let Some(sentinel) = sora_sentinel {
info!("override sentinel value");
credentials.sentinel = Some(sentinel);
}
return Ok(credentials)
},
Err(err) => {
error!("Failed to get Sora credentials from Redis; using env as backup. Error: {:?}", err);
// NB: Fall through
}
};
match SoraCredentials::from_env() {
Ok(creds) => Ok(creds),
Err(err) => {
error!("Failed to load Sora credentials from environment: {:?}", err);
Err(anyhow!("Failed to load Sora credentials from environment"))
}
}
}
@@ -0,0 +1,3 @@
pub mod upload;
pub mod prompt;
pub mod get_sora_credentials_from_request;
@@ -0,0 +1,465 @@
#![forbid(unused_mut)]
use std::fmt::Debug;
use std::fmt::{Display, Formatter};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use log::{debug, error, info};
use log::warn;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use openai_sora_client::credentials::SoraCredentials;
use serde::Deserialize;
use serde::Serialize;
use sqlx::MySqlPool;
use tempdir::TempDir;
use utoipa::ToSchema;
use web::Data;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use http_server_common::request::get_request_ip::get_request_ip;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::{
GenericInferenceArgs,
InferenceCategoryAbbreviated,
PolymorphicInferenceArgs,
};
use mysql_queries::payloads::generic_inference_args::inner_payloads::sora_image_gen_args::SoraImageGenArgs;
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{
insert_generic_inference_job,
InsertGenericInferenceArgs,
};
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use openai_sora_client::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::upload::upload_media_from_file::{sora_media_upload_from_file, SoraMediaUploadRequest};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
use crate::http_server::endpoints::image_studio::get_sora_credentials_from_request::get_sora_credentials_from_request;
use crate::http_server::endpoints::tts::get_tts_inference_job_status::GetTtsInferenceStatusError;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::state::server_state::ServerState;
use crate::util::allowed_studio_access::allowed_studio_access;
use crate::util::redis_sora_secrets::get_sora_credentials;
/// This is the number of images (batch size) to generate for each request.
/// We should allow all users to have multiple images generated at once as this
/// is what other providers do.
const MINIMUM_IMAGE_COUNT : u32 = 1;
/// The maximum number of images (batch size) to generate for each request.
const MAXIMUM_IMAGE_COUNT : u32 = 2;
/// Debug requests can get routed to special "debug-only" workers, which can
/// be used to trial new code, run debugging, etc.
const DEBUG_HEADER_NAME: &str = "enable-debug-mode";
/// The routing tag header can send workloads to particular k8s hosts.
/// This is useful for catching the live logs or intercepting the job.
const ROUTING_TAG_HEADER_NAME: &str = "routing-tag";
#[derive(Deserialize, ToSchema)]
pub struct EnqueueStudioImageGenRequest {
pub uuid_idempotency_token: String,
/// Image media file; the engine or canvas snapshot (screenshot).
pub snapshot_media_token: MediaFileToken,
/// The user's image generation prompt.
pub prompt: String,
/// Additional images to include (optional). Up to nine images.
pub maybe_additional_images: Option<Vec<MediaFileToken>>,
pub maybe_number_of_samples: Option<u32>,
}
#[derive(Serialize, ToSchema)]
pub struct EnqueueImageGenRequestSuccessResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
#[derive(Debug, ToSchema)]
pub enum EnqueueImageGenRequestError {
BadInput(String),
NotAuthorized,
ServerError,
RateLimited,
}
impl ResponseError for EnqueueImageGenRequestError {
fn status_code(&self) -> StatusCode {
match *self {
EnqueueImageGenRequestError::BadInput(_) => StatusCode::BAD_REQUEST,
EnqueueImageGenRequestError::NotAuthorized => StatusCode::UNAUTHORIZED,
EnqueueImageGenRequestError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
EnqueueImageGenRequestError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
EnqueueImageGenRequestError::BadInput(reason) => reason.to_string(),
EnqueueImageGenRequestError::NotAuthorized => "unauthorized".to_string(),
EnqueueImageGenRequestError::ServerError => "server error".to_string(),
EnqueueImageGenRequestError::RateLimited => "rate limited".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
impl Display for EnqueueImageGenRequestError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
/// Prompt image generation using image studio
#[utoipa::path(
post,
tag = "Image Studio",
path = "/v1/image_studio/prompt",
responses(
(
status = 200,
description = "Enqueue TTS generically",
body = EnqueueImageGenRequestSuccessResponse,
),
(status = 400, description = "Bad input", body = EnqueueImageGenRequestError),
(status = 401, description = "Not authorized", body = EnqueueImageGenRequestError),
(status = 429, description = "Rate limited", body = EnqueueImageGenRequestError),
(status = 500, description = "Server error", body = EnqueueImageGenRequestError)
),
params(("request" = EnqueueStudioImageGenRequest, description = "Payload for Image Generation Request"))
)]
pub async fn enqueue_studio_image_generation_handler(
http_request: HttpRequest,
request: Json<EnqueueStudioImageGenRequest>,
server_state: Data<Arc<ServerState>>
) -> Result<HttpResponse, EnqueueImageGenRequestError> {
validate_request(&request)?;
let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| {
warn!("MySql pool error: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
// ==================== USER SESSION ==================== //
let maybe_user_session = server_state.session_checker
.maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection).await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
let mut maybe_user_token: Option<UserToken> = None;
if let Some(user_session) = maybe_user_session.as_ref() {
maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token));
}
// ==================== PLANS ==================== //
// Plan should handle "first anonymous use" and "investor" cases.
let plan = get_correct_plan_for_session(
server_state.server_environment_old,
maybe_user_session.as_ref()
);
// Separate priority for animation.
let priority_level = plan.web_vc_base_priority_level();
// ==================== DEBUG MODE + ROUTING TAG ==================== //
let is_debug_request = get_request_header_optional(&http_request, DEBUG_HEADER_NAME).is_some();
let maybe_routing_tag = get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME).map(
|routing_tag| routing_tag.trim().to_string()
);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.role.is_banned {
return Err(EnqueueImageGenRequestError::NotAuthorized);
}
}
// DETECT premium user and queue
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.logged_out,
Some(ref _user) => &server_state.redis_rate_limiters.logged_in,
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
return Err(EnqueueImageGenRequestError::RateLimited);
}
// Get up IP address
let ip_address = get_request_ip(&http_request);
// ==================== HANDLE IDEMPOTENCY ==================== //
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(EnqueueImageGenRequestError::BadInput(reason));
}
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
EnqueueImageGenRequestError::BadInput("invalid idempotency token".to_string())
})?;
// ==================== DOWNLOAD FILES AND UPLOAD TO SORA ==================== //
// TODO: Maybe this moves to a job.
let work_temp_dir = server_state.temp_dir_creator.new_tempdir("image_studio").unwrap();
let public_bucket_client = server_state.public_bucket_client.clone();
let scene_media_token = request.snapshot_media_token.clone();
let scene_media_path = query_and_download_media_file(
&scene_media_token,
&work_temp_dir,
&public_bucket_client,
&server_state.mysql_pool
).await.map_err(|err| {
error!("Failed to download scene media file: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
let mut files_to_upload = Vec::with_capacity(1 + request.maybe_additional_images
.as_ref().map(|v| v.len()).unwrap_or(0));
files_to_upload.push(scene_media_path.clone());
let additional_images = request.maybe_additional_images.clone().unwrap_or_else(|| vec![]);
for media_file_token in &additional_images {
let media_file_path = query_and_download_media_file(
&media_file_token,
&work_temp_dir,
&public_bucket_client,
&server_state.mysql_pool
).await.map_err(|err| {
error!("Failed to download additional media file: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
files_to_upload.push(media_file_path.clone());
}
// ==================== HANDLE SORA CREDENTIALS ==================== //
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
let sora_credentials = get_sora_credentials_from_request(&http_request, &mut redis)
.map_err(|e| {
error!("sora credential error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
// ==================== HANDLE SORA UPLOAD ==================== //
let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len());
for file_path in files_to_upload {
let scene_media_upload_request = SoraMediaUploadRequest {
file_path: scene_media_path.to_string_lossy().to_string(),
credentials: &sora_credentials,
};
let sora_upload_response = sora_media_upload_from_file(file_path, &sora_credentials)
.await
.map_err(|err| {
error!("Failed to upload scene media to Sora: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
debug!("Uploaded media to Sora : {:?}", sora_upload_response);
sora_media_tokens.push(sora_upload_response.id);
}
// ==================== HANDLE SORA PROMPT ==================== //
let number_of_samples = match request.maybe_number_of_samples {
Some(val) => if val > MAXIMUM_IMAGE_COUNT {
MAXIMUM_IMAGE_COUNT
} else if val < MINIMUM_IMAGE_COUNT {
MINIMUM_IMAGE_COUNT
} else {
val
},
None => 1,
};
let prompt = request.prompt.clone();
let response = sora_image_gen_remix(SoraImageGenRemixRequest {
prompt: prompt.clone(),
num_images: NumImages::One,
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: &sora_credentials,
}).await
.map_err(|err| {
error!("Failed to call Sora image generation: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
debug!("Sora image gen response: {:?}", response);
// Store the actual inference args that will go to the database
let inference_args = SoraImageGenArgs {
prompt: Some(prompt),
scene_snapshot_media_token: Some(request.snapshot_media_token.clone()),
maybe_additional_media_file_tokens: Some(additional_images),
maybe_number_of_samples: Some(number_of_samples),
maybe_sora_media_upload_tokens: Some(sora_media_tokens),
};
// create the inference args here
let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request);
// create the job record here!
let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
job_type: InferenceJobType::ImageGenApi,
maybe_product_category: None, // This is not a product anymore
inference_category: InferenceCategory::ImageGeneration,
maybe_model_type: Some(InferenceModelType::ImageGenApi), // NB: Model is static during inference
maybe_model_token: None, // NB: Model is static during inference
maybe_input_source_token: None,
maybe_input_source_token_type: None,
maybe_download_url: None,
maybe_cover_image_media_file_token: None,
maybe_raw_inference_text: None,
maybe_max_duration_seconds: None,
maybe_inference_args: Some(GenericInferenceArgs {
inference_category: Some(InferenceCategoryAbbreviated::ImageGeneration),
args: Some(PolymorphicInferenceArgs::Sg(inference_args)),
}),
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
priority_level,
requires_keepalive: false, //reverse ... TODO fix this. we set it base on account is premium or not ...
is_debug_request,
maybe_routing_tag: maybe_routing_tag.as_deref(),
mysql_pool: &server_state.mysql_pool,
}).await;
let job_token = match query_result {
Ok((job_token, _id)) => job_token,
Err(err) => {
warn!("New generic inference job creation DB error: {:?}", err);
if err.had_duplicate_idempotency_token() {
return Err(EnqueueImageGenRequestError::BadInput("Duplicate idempotency token".to_string()));
}
return Err(EnqueueImageGenRequestError::ServerError);
}
};
let response: EnqueueImageGenRequestSuccessResponse = EnqueueImageGenRequestSuccessResponse {
success: true,
inference_job_token: job_token,
};
let body = serde_json::to_string(&response)
.map_err(|_e| EnqueueImageGenRequestError::ServerError)?;
// Error handling 101 rust result type returned like so.
Ok(HttpResponse::Ok().content_type("application/json").body(body))
}
fn validate_request(
request: &Json<EnqueueStudioImageGenRequest>,
) -> Result<(), EnqueueImageGenRequestError> {
Ok(())
}
#[derive(Debug)]
pub enum DownloadMediaFileError {
IoError(std::io::Error),
MediaFileNotFound,
Other(anyhow::Error),
}
impl From<anyhow::Error> for DownloadMediaFileError {
fn from(err: anyhow::Error) -> Self {
DownloadMediaFileError::Other(err)
}
}
async fn query_and_download_media_file(
media_file_token: &MediaFileToken,
download_dir: &TempDir,
bucket_client: &BucketClient,
mysql_pool: &MySqlPool,
) -> Result<PathBuf, DownloadMediaFileError> {
let media_file_result = get_media_file(
&media_file_token,
false,
mysql_pool
).await;
let media_file= match media_file_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media file in database: {:?}", media_file_token);
return Err(DownloadMediaFileError::MediaFileNotFound)
}
Err(e) => {
error!("could not query media file: {:?}", e);
return Err(DownloadMediaFileError::Other(e))
}
};
let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
&media_file.public_bucket_directory_hash,
media_file.maybe_public_bucket_prefix.as_deref(),
media_file.maybe_public_bucket_extension.as_deref());
let extension = media_file.maybe_public_bucket_extension.clone().unwrap_or_else(|| ".jpg".to_string());
let download_filename = format!("download_file_{}.{}", media_file.token.as_str(), extension);
let download_file_path = download_dir.path().join(download_filename);
info!("Downloading from bucket...");
bucket_client.download_file_to_disk(
&media_file_bucket_path.to_full_object_pathbuf(),
&download_file_path,
).await?;
Ok(download_file_path)
}
@@ -0,0 +1 @@
pub mod enqueue_studio_image_generation_handler;
@@ -0,0 +1 @@
pub mod upload_snapshot_media_file_handler;
@@ -0,0 +1,279 @@
use std::collections::HashSet;
use std::io::{BufReader, Bytes, Cursor, Read};
use std::path::PathBuf;
use std::sync::Arc;
use actix_multipart::form::tempfile::TempFile;
use actix_multipart::form::text::Text;
use actix_multipart::form::MultipartForm;
use actix_multipart::Multipart;
use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use log::{error, info, warn};
use once_cell::sync::Lazy;
use utoipa::ToSchema;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mimetypes::mimetype_to_extension::mimetype_to_extension;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult};
/// Form-multipart request fields.
///
/// IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `UploadSnapshotMediaFileForm` (Under "Schema") FOR DETAILS ON FIELDS AND NULLABILITY.
#[derive(MultipartForm, ToSchema)]
#[multipart(duplicate_field = "deny")]
pub struct UploadSnapshotMediaFileForm {
/// UUID for request idempotency
#[multipart(limit = "2 KiB")]
#[schema(value_type = String, format = Binary)]
uuid_idempotency_token: Text<String>,
/// The uploaded "screenshot" / "snapshot" of the engine scene
#[multipart(limit = "512 MiB")]
#[schema(value_type = Vec<u8>, format = Binary)]
snapshot: TempFile,
/// Optional: If an engine scene was used to create this snapshot, provide it here to create a link
#[multipart(limit = "2 KiB")]
#[schema(value_type = Option<MediaFileToken>, format = Binary)]
scene_media_token: Option<Text<MediaFileToken>>,
/// Optional: Visibility of the scene
#[multipart(limit = "2 KiB")]
#[schema(value_type = Option<Visibility>, format = Binary)]
maybe_visibility: Option<Text<Visibility>>,
}
// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*.
#[derive(Serialize, ToSchema)]
pub struct UploadSnapshotMediaFileSuccessResponse {
pub success: bool,
/// The media token of the "screenshot" / "snapshot" we took.
pub snapshot_media_token: MediaFileToken,
}
static ALLOWED_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
HashSet::from([
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
])
});
/// Upload snapshots of the scenes created in the 3D image studio.
#[utoipa::path(
post,
tag = "Image Studio",
path = "/v1/image_studio/scene_snapshot",
responses(
(status = 200, description = "Success Update", body = UploadSnapshotMediaFileSuccessResponse),
(status = 400, description = "Bad input", body = MediaFileUploadError),
(status = 401, description = "Not authorized", body = MediaFileUploadError),
(status = 429, description = "Too many requests", body = MediaFileUploadError),
(status = 500, description = "Server error", body = MediaFileUploadError),
),
params(
(
"request" = UploadSnapshotMediaFileForm,
description = "IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `UploadSnapshotMediaFileForm` (Under 'Schema') FOR DETAILS ON FIELDS AND NULLABILITY."
),
)
)]
pub async fn upload_snapshot_media_file_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
MultipartForm(mut form): MultipartForm<UploadSnapshotMediaFileForm>,
) -> Result<Json<UploadSnapshotMediaFileSuccessResponse>, MediaFileUploadError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await
.map_err(|err| {
error!("MySql pool error: {:?}", err);
MediaFileUploadError::ServerError
})?;
// ==================== READ SESSION ==================== //
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
error!("Session checker error: {:?}", e);
MediaFileUploadError::ServerError
})?;
let maybe_user_token = maybe_user_session
.as_ref()
.map(|session| session.get_strongly_typed_user_token());
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.is_banned {
return Err(MediaFileUploadError::NotAuthorizedVerbose("user is banned".to_string()));
}
}
// ==================== HANDLE IDEMPOTENCY ==================== //
// TODO(bt, 2024-02-26): This should be a transaction.
let uuid_idempotency_token = form.uuid_idempotency_token.as_ref();
if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) {
return Err(MediaFileUploadError::BadInput(reason));
}
insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
MediaFileUploadError::BadInput("invalid idempotency token".to_string())
})?;
// ==================== UPLOAD METADATA ==================== //
let creator_set_visibility = form.maybe_visibility
.map(|visibility| visibility.0)
.or_else(|| {
maybe_user_session
.as_ref()
.map(|user_session| user_session.preferred_tts_result_visibility)
})
.unwrap_or(Visibility::default());
// ==================== USER DATA ==================== //
let ip_address = get_request_ip(&http_request);
// ==================== FILE VALIDATION ==================== //
let mut file_bytes = Vec::new();
form.snapshot.file.read_to_end(&mut file_bytes)
.map_err(|e| {
error!("Problem reading file: {:?}", e);
MediaFileUploadError::ServerError
})?;
let mimetype = get_mimetype_for_bytes(file_bytes.as_ref())
.map(|mimetype| mimetype.to_string())
.ok_or_else(|| {
warn!("Could not determine mimetype for file");
MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string())
})?;
if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) {
// NB: Don't let our error message inject malicious strings
let filtered_mimetype = mimetype
.chars()
.filter(|c| c.is_ascii())
.filter(|c| c.is_alphanumeric() || *c == '/')
.collect::<String>();
return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype)));
}
// ==================== OTHER FILE METADATA ==================== //
let maybe_filename = form.snapshot.file_name.as_deref()
.as_deref()
.map(|filename| PathBuf::from(filename));
let extension = mimetype_to_extension(&mimetype)
.or_else(|| {
maybe_filename
.as_ref()
.and_then(|filename| filename.extension()) // TODO needs dot prefix
.and_then(|ext| ext.to_str())
})
.ok_or_else(|| {
warn!("Could not determine file extension for mimetype: {}", &mimetype);
MediaFileUploadError::ServerError
})?;
let extension = format!(".{extension}"); // NB: needs dot prefix
let file_size_bytes = file_bytes.len();
let hash = sha256_hash_bytes(&file_bytes)
.map_err(|io_error| {
error!("Problem hashing bytes: {:?}", io_error);
MediaFileUploadError::ServerError
})?;
// ==================== UPLOAD AND SAVE ==================== //
const PREFIX : Option<&str> = Some("snapshot_");
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension));
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
server_state.public_bucket_client.upload_file_with_content_type(
public_upload_path.get_full_object_path_str(),
file_bytes.as_ref(),
&mimetype)
.await
.map_err(|e| {
warn!("Upload media bytes to bucket error: {:?}", e);
MediaFileUploadError::ServerError
})?;
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Image),
media_file_type: MediaFileType::Image,
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility,
upload_type: UploadType::Filesystem, // TODO(bt,2024-05-02): This should be a parameter and a well-known enum.
maybe_engine_category: None,
maybe_animation_type: None,
maybe_mime_type: Some(&mimetype),
file_size_bytes: file_size_bytes as u64,
maybe_duration_millis: None,
sha256_checksum: &hash,
maybe_title: Some("Snapshot"),
maybe_scene_source_media_file_token: None,
is_intermediate_system_file: true,
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: Some(&extension),
pool: &server_state.mysql_pool,
})
.await
.map_err(|err| {
warn!("New file creation DB error: {:?}", err);
MediaFileUploadError::ServerError
})?;
info!("new media file id: {} token: {:?}", record_id, &token);
Ok(Json(UploadSnapshotMediaFileSuccessResponse {
success: true,
snapshot_media_token: token,
}))
}
@@ -5,6 +5,7 @@ pub mod app_state;
pub mod beta_keys;
pub mod categories;
pub mod comments;
pub mod control_plane;
pub mod conversion;
pub mod download_job;
pub mod engine;
@@ -12,6 +13,7 @@ pub mod events;
pub mod featured_items;
pub mod flags;
pub mod image_gen;
pub mod image_studio;
pub mod inference_job;
pub mod investor_demo;
pub mod leaderboard;
@@ -30,9 +32,9 @@ pub mod subscriptions;
pub mod tags;
pub mod trending;
pub mod tts;
pub mod users;
pub mod user_bookmarks;
pub mod user_ratings;
pub mod users;
pub mod vocoders;
pub mod voice_clone_requests;
pub mod voice_conversion;
@@ -0,0 +1,24 @@
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
use crate::http_server::endpoints::control_plane::set_sora_secret_handler::set_sora_secret_handler;
pub fn add_control_plane_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/control_plane")
.service(web::resource("/sora_secret")
.route(web::post().to(set_sora_secret_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
}
@@ -0,0 +1,29 @@
use crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::enqueue_studio_image_generation_handler;
use crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::upload_snapshot_media_file_handler;
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
pub fn add_image_studio_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/image_studio")
.service(web::resource("/scene_snapshot")
.route(web::post().to(upload_snapshot_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/prompt")
.route(web::post().to(enqueue_studio_image_generation_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
}
@@ -123,6 +123,8 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp
use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler;
use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler;
use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler;
use crate::http_server::routes::add_control_plane_routes::add_control_plane_routes;
use crate::http_server::routes::add_image_studio_routes::add_image_studio_routes;
use crate::http_server::routes::add_studio_gen2_routes::add_studio_gen2_routes;
use crate::http_server::routes::beta_key_routes::add_beta_key_routes;
use crate::http_server::routes::job_routes::add_job_routes;
@@ -171,8 +173,10 @@ pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) ->
app = add_model_download_routes(app);
app = add_workflow_routes(app);
app = add_studio_gen2_routes(app);
app = add_image_studio_routes(app);
app = add_job_routes(app);
app = add_engine_routes(app); /* /v1/engine/... */
app = add_control_plane_routes(app); /* /v1/control_plane/... */
// app = add_image_gen_routes(app);
@@ -8,4 +8,6 @@ pub mod moderation_routes;
pub mod user_routes;
pub mod weights_routes;
pub mod workflow_routes;
mod tag_routes;
mod tag_routes;
mod add_image_studio_routes;
mod add_control_plane_routes;
@@ -11,6 +11,20 @@ pub struct ScopedTempDirCreator {
}
impl ScopedTempDirCreator {
pub fn auto_setup() -> Self {
// Production configuration
let maybe_tmp = easyenv::get_env_pathbuf_optional("TEMP_DIR");
// MacOS
// TODO: Only run this on macOS
let maybe_mac_tmp = easyenv::get_env_pathbuf_optional("TMPDIR");
let directory = maybe_tmp
.or(maybe_mac_tmp)
.unwrap_or_else(|| PathBuf::from("/tmp"));
Self::for_directory(directory)
}
pub fn for_directory<P: AsRef<Path>>(base_dir: P) -> Self {
Self {
@@ -464,7 +464,7 @@ async fn main() -> AnyhowResult<()> {
user_tokens: user_token_troll_bans,
ip_addresses: ip_address_troll_bans,
},
temp_dir_creator: ScopedTempDirCreator::for_directory(easyenv::get_env_pathbuf_or_default("TEMP_DIR", "/tmp")),
temp_dir_creator: ScopedTempDirCreator::auto_setup(),
google_sign_in_cert: GoogleSignInCert::new(),
};
@@ -11,5 +11,6 @@ pub mod enroll_in_studio;
pub mod generate_random_username;
pub mod placeholder_images;
pub mod read_toml_file_to_struct;
pub mod redis_sora_secrets;
pub mod title_to_url_slug;
pub mod troll_user_bans;
@@ -0,0 +1,67 @@
use std::collections::HashMap;
use anyhow::anyhow;
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::redis::Commands;
use r2d2_redis::RedisConnectionManager;
use errors::AnyhowResult;
use openai_sora_client::credentials::SoraCredentials;
/// Redis key
const SORA_SECRET_REDIS_KEY : &str = "sora_secret";
// Fields within the HKEY
const BEARER_SUBKEY : &str = "bearer";
const COOKIE_SUBKEY : &str = "cookie";
const SENTINEL_SUBKEY : &str = "sentinel";
#[derive(Clone,Copy,Debug)]
pub enum RedisSoraCredentialSubkey {
Bearer,
Cookie,
Sentinel,
}
impl RedisSoraCredentialSubkey {
pub fn to_str(&self) -> &'static str {
match self {
RedisSoraCredentialSubkey::Bearer => BEARER_SUBKEY,
RedisSoraCredentialSubkey::Cookie => COOKIE_SUBKEY,
RedisSoraCredentialSubkey::Sentinel => SENTINEL_SUBKEY,
}
}
}
pub fn get_sora_credentials(
redis: &mut PooledConnection<RedisConnectionManager>
) -> AnyhowResult<SoraCredentials> {
let values : HashMap<String, String> = redis.hgetall(SORA_SECRET_REDIS_KEY)
.map_err(|e| anyhow!("Failed to get Sora credentials from Redis: {}", e))?;
let bearer = values.get(BEARER_SUBKEY);
let cookie = values.get(COOKIE_SUBKEY);
let sentinel = values.get(SENTINEL_SUBKEY);
match (bearer, cookie, sentinel) {
(Some(b), Some(c), Some(s)) => {
Ok(SoraCredentials {
bearer_token: b.to_string(),
cookie: c.to_string(),
sentinel: Some(s.to_string()),
})
}
_ => Err(anyhow!("redis sora credential values not present")),
}
}
pub fn set_sora_credential_field(
redis: &mut PooledConnection<RedisConnectionManager>,
subkey: RedisSoraCredentialSubkey,
value: &str,
) -> AnyhowResult<()> {
redis.hset(SORA_SECRET_REDIS_KEY, subkey.to_str(), value)
.map_err(|e| anyhow!("Failed to set Sora credentials in Redis: {}", e))?;
Ok(())
}
@@ -36,4 +36,174 @@ export class EngineApi extends ApiManager {
};
});
}
public async uploadSceneSnapshot({
screenshot,
sceneMediaToken,
}: {
screenshot: File; // base64 encoded PNG
sceneMediaToken?: string;
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
const formData = new FormData();
formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
if (sceneMediaToken) {
formData.append("scene_media_token", sceneMediaToken);
}
// for now ...
const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token
const response = await fetch(endpoint, {
method: "POST",
headers: {
Accept: "application/json",
},
credentials: "include",
body: formData,
});
const postResponse = await response.json();
console.log(postResponse);
let result: { success: boolean; data?: string; errorMessage?: string };
if (postResponse.success) {
result = {
success: true,
data: postResponse.snapshot_media_token,
errorMessage: undefined,
};
} else {
result = {
success: false,
errorMessage: postResponse.BadInput,
};
}
return result;
}
public async enqueueImageGeneration({
prompt,
snapshotMediaToken,
additionalImages,
}: {
prompt: string;
snapshotMediaToken: string;
additionalImages?: string[];
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
// for now ...
const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
const body = {
uuid_idempotency_token: uuidIdempotencyToken,
prompt,
snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
additional_images: additionalImages,
};
const postResponse = await this.post<
{
uuid_idempotency_token: string;
prompt: string;
snapshot_media_token: string;
additional_images?: string[];
},
{ success?: boolean; job_token?: string; BadInput?: string }
>({
endpoint,
body,
});
// Check if the response is successful
const isSuccess = postResponse.success ?? false;
// Prepare the result object
const result = {
success: isSuccess,
data: isSuccess ? postResponse.job_token : undefined,
errorMessage: isSuccess ? undefined : postResponse.BadInput,
};
return result;
}
public async pollJobSession(jobToken: string): Promise<
ApiResponse<{
status: string;
result: {
generated_images?: string[];
error?: string;
};
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`;
const response = await this.get<{
success?: boolean;
status?: string;
result?: {
generated_images?: string[];
error?: string;
};
BadInput?: string;
}>({
endpoint,
});
const success = response.success ?? false;
const status = response.status ?? "";
const result = response.result ?? {
generated_images: [],
error: undefined,
};
const errorMessage = response.BadInput;
return {
success,
data: { status, result },
errorMessage,
};
}
public async pollStudioSessionJobs(jobToken: string): Promise<
ApiResponse<{
status: string;
result: {
generated_images?: string[];
error?: string;
};
}>
> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`;
const response = await this.get<{
success?: boolean;
status?: string;
result?: {
generated_images?: string[];
error?: string;
};
BadInput?: string;
}>({
endpoint,
});
const success = response.success ?? false;
const status = response.status ?? "";
const result = response.result ?? {
generated_images: [],
error: undefined,
};
const errorMessage = response.BadInput;
return {
success,
data: { status, result },
errorMessage,
};
}
}
@@ -15,6 +15,8 @@ export const uploadImage = async ({
}) => {
const mediaUploadApi = new MediaUploadApi();
console.log("Uploading image:", assetFile);
progressCallback({ status: UploaderStates.uploadingImage });
const imageResponse = await mediaUploadApi.UploadImage({
@@ -24,7 +26,10 @@ export const uploadImage = async ({
maybe_title: "char_frame_" + title,
});
console.log("Image response:", imageResponse);
if (imageResponse == undefined) {
console.log("Error: Could not upload image!");
progressCallback({
status: UploaderStates.imageCreateError,
errorMessage: "Could not upload image!",
@@ -33,6 +38,7 @@ export const uploadImage = async ({
}
if (!imageResponse.success || !imageResponse.data) {
console.log("Error:", imageResponse.errorMessage);
progressCallback({
status: UploaderStates.imageCreateError,
errorMessage: imageResponse.errorMessage,
@@ -40,5 +46,9 @@ export const uploadImage = async ({
return;
}
progressCallback({ status: UploaderStates.success, data: imageResponse.data });
console.log("Upload successful:", imageResponse.data);
progressCallback({
status: UploaderStates.success,
data: imageResponse.data,
});
};
@@ -0,0 +1,146 @@
import { TransformControls } from "three/examples/jsm/Addons";
import * as THREE from "three";
export const FKHelperSphereName = "__FKHelperSphere__";
export class FKHelper {
private targetBoneSpheres: THREE.Object3D<THREE.Object3DEventMap>[];
private transformControls: TransformControls;
private raycaster: THREE.Raycaster;
private scene: THREE.Scene;
private skeletonHelper: THREE.SkeletonHelper | null = null;
private isControlInScene = false;
// Need the actual object to perform FK on
// This means the object must have bones
// Not all bones require FK right now
// Need a set of names to filter bones
// Use mixamo names
// Need a transform control (rotation mode) for the bones
// Need to disable the object transforms for bone transforms
// Need raycaster
constructor({
camera,
domElement,
scene,
onDragChange
}: {
camera: THREE.Camera;
domElement: HTMLElement;
scene: THREE.Scene;
onDragChange: (dragging: boolean) => void;
}) {
this.raycaster = new THREE.Raycaster();
this.scene = scene;
this.targetBoneSpheres = [];
this.transformControls = new TransformControls(camera, domElement);
this.transformControls.setSpace("local");
this.transformControls.setMode("rotate");
this.transformControls.setSize(0.5);
this.transformControls.addEventListener("dragging-changed", (event: any) => {
onDragChange(event.value);
});
}
// Set FK target
setTarget(target: THREE.Object3D) {
// TODO: Highlight all the necessary bones
this.clear();
target.traverse((child) => {
if (child.type !== "Bone") {
return;
}
// For each target bone, display a sphere to show the FK target and raycast intersect
const geometry = new THREE.SphereGeometry(1, 16, 16);
const material = new THREE.MeshBasicMaterial({ color: 0xff0000, depthTest: false, transparent: true });
const sphere = new THREE.Mesh(geometry, material);
sphere.name = FKHelperSphereName;
const uniformScale = 1.5;
sphere.scale.set(uniformScale, uniformScale, uniformScale);
child.add(sphere);
this.targetBoneSpheres.push(sphere);
});
this.skeletonHelper = new THREE.SkeletonHelper(target);
this.scene.add(this.skeletonHelper);
}
onMouseClick(mouse: THREE.Vector2) {
this.raycaster.setFromCamera(mouse, this.transformControls.camera);
const intersections = this.raycaster.intersectObjects(this.targetBoneSpheres, false);
if (intersections.length < 1) {
console.log("No intersections");
this.resetHighlight();
this.removeControls();
return;
}
// Attach the controls to the first bone
const firstBone = intersections[0];
this.highlightSphere(firstBone.object);
this.showControls(firstBone.object.parent!);
console.log("Showing bone", firstBone.object);
}
private showControls(object?: THREE.Object3D) {
if (object) {
this.transformControls.attach(object);
}
if (this.isControlInScene) {
return;
}
this.scene.add(this.transformControls);
this.isControlInScene = true;
}
private highlightSphere(sphere: THREE.Object3D) {
this.targetBoneSpheres.forEach((s) => {
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
s.material.opacity = 0.2
});
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
sphere.material.opacity = 1;
}
private resetHighlight() {
this.targetBoneSpheres.forEach((s) => {
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
s.material.opacity = 1;
});
}
private removeControls() {
this.transformControls.detach();
if (this.isControlInScene) {
this.scene.remove(this.transformControls);
this.isControlInScene = false;
}
}
private removeSpheres() {
this.targetBoneSpheres.forEach((sphere) => {
this.scene.remove(sphere);
sphere.parent?.remove(sphere);
// @ts-expect-error Geometry on object3d, but it's fine since this is a mesh
sphere.geometry.dispose();
});
this.targetBoneSpheres = [];
}
clear() {
this.removeControls();
this.removeSpheres();
if (this.skeletonHelper) {
this.scene.remove(this.skeletonHelper!);
}
}
}
@@ -655,6 +655,35 @@ class Editor {
document.body.appendChild(this.stats.dom);
}
public snapShotOfCurrentFrame(shouldDownload: boolean = true) {
const snapshot = this.renderer?.domElement.toDataURL("image/png");
if (!snapshot) {
console.error("Error: Snapshot could not be created.");
return null;
}
const base64Snapshot = snapshot.split(",")[1];
if (shouldDownload) {
const link = document.createElement("a");
link.download = "scene-snapshot.png";
link.href = snapshot;
link.click();
}
const byteString = atob(base64Snapshot);
const mimeString = "image/png";
const ab = new ArrayBuffer(byteString.length);
const ia = new Uint8Array(ab);
for (let i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
const uuid = crypto.randomUUID(); // Generate a new UUID
const file = new File([ab], `${uuid}.png`, { type: mimeString });
return { base64Snapshot, file };
}
public async newScene(sceneTitleInput: string) {
this.activeScene.clear();
this.audio_engine = new AudioEngine();
@@ -931,6 +960,8 @@ class Editor {
}
deleteObject(uuid: string) {
this.mouse_controls?.clearFKVisuals();
this.mouse_controls?.removeTransformControls(true);
this.utils.deleteObject(uuid);
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
@@ -962,26 +993,26 @@ class Editor {
this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
? 1024
: this.render_camera_aspect_ratio ===
CameraAspectRatio.VERTICAL_9_16
meraAspectRatio.VERTICAL_9_16
? 576
: this.render_camera_aspect_ratio ===
CameraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
CameraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
? 576
: this.render_camera_aspect_ratio ===
CameraAspectRatio.VERTICAL_9_16
meraAspectRatio.VERTICAL_9_16
? 1024
: this.render_camera_aspect_ratio ===
CameraAspectRatio.HORIZONTAL_3_2
meraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
CameraAspectRatio.VERTICAL_2_3
meraAspectRatio.VERTICAL_2_3
? 900
: 1000;
@@ -14,6 +14,13 @@ import { TransformControls } from "./TransformControls";
import { SceneManager, SceneObject } from "./scene_manager_api";
import { FreeCam } from "./free_cam";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { FKHelper } from "./KinHelpers/FKHelper";
export enum KinMode {
FK,
IK,
NONE,
}
export class MouseControls {
camera: THREE.PerspectiveCamera | null;
@@ -47,8 +54,12 @@ export class MouseControls {
private isMovable: Function;
enable_stats: Function;
private kinMode: KinMode = KinMode.NONE;
private fkHelper: FKHelper;
private isBoneDragged: boolean = false;
constructor(
camera: THREE.PerspectiveCamera | null,
camera: THREE.PerspectiveCamera,
camera_person_mode: boolean,
cameraViewControls: FreeCam,
lockControls: PointerLockControls | undefined,
@@ -62,7 +73,7 @@ export class MouseControls {
mouse: THREE.Vector2 | undefined,
timeline_mouse: THREE.Vector2 | undefined,
raycaster: THREE.Raycaster | undefined,
control: TransformControls | undefined,
control: TransformControls,
outlinePass: OutlinePass | undefined,
scene: THREE.Scene,
publishSelect: Function,
@@ -101,6 +112,20 @@ export class MouseControls {
this.sceneManager = undefined;
this.isMovable = isMovable;
this.enable_stats = enable_stats;
this.fkHelper = new FKHelper({
camera: this.camera,
domElement: this.control.domElement,
scene: this.scene,
onDragChange: this.onFKControlsDragging.bind(this),
});
}
onFKControlsDragging(dragging: boolean) {
this.isBoneDragged = dragging;
}
clearFKVisuals() {
this.fkHelper.clear();
}
focus() {
@@ -125,9 +150,26 @@ export class MouseControls {
this.selected = [];
this.publishSelect();
}
this.hideTransformControls();
if (remove_outline) this.outlinePass.selectedObjects = [];
}
hideTransformControls() {
if (this.control == undefined) {
return;
}
this.control.detach();
this.scene.remove(this.control);
if (remove_outline) this.outlinePass.selectedObjects = [];
}
reattachTransformControls() {
if (this.control == undefined || this.selected == undefined) {
return;
}
this.control.attach(this.selected[0]);
this.scene.add(this.control);
}
selectObject(currentObject: THREE.Object3D) {
@@ -151,7 +193,7 @@ export class MouseControls {
}
onMouseDown(event: any) {
if ((event.button === 0 || event.button === 1) && this.isMovable()) {
if ((event.button === 0 || event.button === 1) && this.isMovable() && !this.isBoneDragged) {
this.isMouseClicked = true;
}
}
@@ -176,6 +218,35 @@ export class MouseControls {
return new Promise((resolve) => setTimeout(resolve, ms));
}
toggleFKMode() {
if (this.kinMode == KinMode.FK) {
this.fkHelper.clear();
this.kinMode = KinMode.NONE;
this.reattachTransformControls();
console.log("FK mode off");
return;
}
// Make sure we have an intersection
if (!(this.selected && this.selected.length > 0)) {
return;
}
// Make sure FK is supported only on character type objects
const firstSelection = this.selected[0];
if (!firstSelection.userData.isCharacter) {
return;
}
// FK is good to go
// Disable main transform controls
this.hideTransformControls();
this.kinMode = KinMode.FK;
this.fkHelper.setTarget(this.selected[0]);
console.log("FK mode on");
return;
}
async onkeydown(event: KeyboardEvent) {
if (hotkeysStatus.value.disabled) {
return;
@@ -198,7 +269,7 @@ export class MouseControls {
// transform
this.control?.setMode("translate");
return;
} else if (event.key === "r") {
} else if (event.key === "r" && !event.ctrlKey) {
// rotate
this.control?.setMode("rotate");
return;
@@ -206,6 +277,8 @@ export class MouseControls {
// scale
this.control?.setMode("scale");
return;
} else if (event.key === "k") {
this.toggleFKMode();
}
if ((event.ctrlKey || event.metaKey) && !this.isProcessing) {
@@ -298,6 +371,11 @@ export class MouseControls {
}
this.camera_last_pos.copy(camera_pos);
if (this.kinMode == KinMode.FK) {
this.fkHelper.onMouseClick(this.mouse);
return;
}
this.raycaster.setFromCamera(this.mouse, this.camera);
const interactable: any[] = [];
this.scene.children.forEach((child: THREE.Object3D) => {
@@ -287,10 +287,6 @@ export class TimeLine {
// testDeformBody(lastRig);
}
// Add a skeleton helper to the character
const skeletonHelper = new THREE.SkeletonHelper(obj);
this.scene.scene.add(skeletonHelper);
const object_uuid = obj.uuid;
this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.
@@ -1,6 +1,8 @@
import { useState, useRef, useEffect } from "react";
import { useSignals } from "@preact/signals-react/runtime";
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
import { useContext } from "react";
import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
import {
faPlus,
faCamera,
@@ -18,11 +20,12 @@ import {
faSquare,
faRectangle,
} from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
import { Button } from "~/components";
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
import { cameraAspectRatio } from "~/pages/PageEnigma/signals";
import { cameraAspectRatio, disableHotkeyInput, DomLevels, enableHotkeyInput } from "~/pages/PageEnigma/signals";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
@@ -31,10 +34,15 @@ interface ReferenceImage {
id: string;
url: string;
file: File;
mediaToken: string;
}
import { EngineApi } from "~/Classes/ApiManager/EngineApi";
import { scene } from "~/signals/scene";
import { UploaderStates } from "~/enums/UploaderStates";
export const PromptBox = () => {
useSignals();
const editorEngine = useContext(EngineContext);
const [prompt, setPrompt] = useState("");
const [isEnqueueing, setisEnqueueing] = useState(false);
const [referenceImages, setReferenceImages] = useState<ReferenceImage[]>([]);
@@ -163,15 +171,27 @@ export const PromptBox = () => {
Array.from(files).forEach((file) => {
const reader = new FileReader();
reader.onloadend = () => {
setReferenceImages((prev) => [
...prev,
{
id: Math.random().toString(36).substring(7),
url: reader.result as string,
file,
uploadImage({
title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
assetFile: file,
progressCallback: (newState) => {
console.debug("Upload progress:", newState.data);
if (newState.status === UploaderStates.success && newState.data) {
const referenceImage: ReferenceImage = {
id: Math.random().toString(36).substring(7),
url: reader.result as string,
file,
mediaToken: newState.data || "",
};
setReferenceImages((prev) => [...prev, referenceImage]);
console.log("Reference image added:", referenceImage);
} else {
console.error("No data from uploadImage");
}
},
]);
});
};
reader.readAsDataURL(file);
});
}
@@ -218,18 +238,37 @@ export const PromptBox = () => {
const handleEnqueue = async () => {
if (!prompt.trim()) return;
setisEnqueueing(true);
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
);
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
setisEnqueueing(false);
if (editorEngine) {
editorEngine.positive_prompt = prompt;
const engineApi = new EngineApi();
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const snapshotResult = await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file,
sceneMediaToken: "",
});
await engineApi.enqueueImageGeneration({
prompt: prompt,
snapshotMediaToken: snapshotResult.data || "",
additionalImages: referenceImages.map((image) => image.mediaToken),
});
}
setisEnqueueing(true);
try {
// Here we would pass both the prompt and reference images to the generation
console.log(
"Enqueuing with prompt:",
prompt,
"and reference images:",
referenceImages,
);
await new Promise((resolve) => setTimeout(resolve, 2000));
} finally {
setisEnqueueing(false);
}
}
};
@@ -258,6 +297,25 @@ export const PromptBox = () => {
}
};
const handleSaveFrame = async () => {
if (editorEngine) {
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const engineApi = new EngineApi();
await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file || "",
sceneMediaToken: "",
});
}
}
};
const handleDownloadFrame = () => {
if (editorEngine) {
editorEngine.snapShotOfCurrentFrame(true);
}
};
return (
<div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 flex-col gap-3">
{referenceImages.length > 0 && (
@@ -325,6 +383,14 @@ export const PromptBox = () => {
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => {
console.log("disabling hotkeys")
disableHotkeyInput(DomLevels.INPUT);
}}
onBlur={() => {
console.log("enabling hotkeys")
enableHotkeyInput(DomLevels.INPUT);
}}
/>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
@@ -360,6 +426,7 @@ export const PromptBox = () => {
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faDownload}
onClick={handleDownloadFrame}
>
Download frame
</Button>
@@ -367,6 +434,7 @@ export const PromptBox = () => {
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
variant="secondary"
icon={faSave}
onClick={handleSaveFrame}
>
Save frame
</Button>
@@ -51,7 +51,7 @@ export const Letterbox = ({
<div
id="letterbox"
className={twMerge(
"absolute left-0 top-0 flex h-full w-full justify-between",
"absolute left-0 top-0 flex h-full w-full justify-between pointer-events-none",
matteOri === MatteOrientation.TOP_BOTTOM ? "flex-col" : null,
)}
>