Compare commits

..
Author SHA1 Message Date
Michael Chung 1ce9c8bed8 Added reference images to the generation
enqueueImageGeneration now accepts additionalImages
enqueueImageGeneration needs testing.
Polling for jobs doesn't work,
most api calls don't work and need integration testing.
2025-04-03 17:26:52 -04:00
Michael Chung 25c15bffd8 Merge branch 'main' into feature/gen 2025-04-03 12:37:31 -04:00
Michael Chung 2d361c6636 Merge branch 'main' into feature/gen 2025-04-03 11:55:38 -04:00
Michael Chung 534afbbbf9 Wire screenshot upload and generate to the engine. 2025-04-02 21:36:42 -04:00
52 changed files with 99 additions and 2533 deletions
Generated
+44 -341
View File
@@ -646,9 +646,9 @@ checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e"
[[package]]
name = "anyhow"
version = "1.0.97"
version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
dependencies = [
"backtrace",
]
@@ -680,7 +680,7 @@ dependencies = [
"ort",
"primitives",
"rand 0.8.5",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_json",
"serde_yaml",
@@ -2399,29 +2399,10 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9"
dependencies = [
"cookie 0.18.1",
"document-features",
"idna 1.0.3",
"log",
"publicsuffix",
"serde",
"serde_derive",
"serde_json",
"time",
"url",
]
[[package]]
name = "cookies"
version = "0.0.1"
@@ -3359,15 +3340,6 @@ dependencies = [
"const-random",
]
[[package]]
name = "document-features"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d"
dependencies = [
"litrs",
]
[[package]]
name = "dotenv"
version = "0.15.0"
@@ -4864,7 +4836,7 @@ dependencies = [
"errors",
"jwt-simple",
"my-workspace-hack",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_derive",
"serde_json",
@@ -5090,7 +5062,7 @@ dependencies = [
"native-tls",
"num_cpus",
"rand 0.8.5",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_json",
"thiserror 2.0.11",
@@ -5465,124 +5437,6 @@ dependencies = [
"png",
]
[[package]]
name = "icu_collections"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
dependencies = [
"displaydoc",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_locid"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
dependencies = [
"displaydoc",
"litemap",
"tinystr",
"writeable",
"zerovec",
]
[[package]]
name = "icu_locid_transform"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
dependencies = [
"displaydoc",
"icu_locid",
"icu_locid_transform_data",
"icu_provider",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_locid_transform_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
[[package]]
name = "icu_normalizer"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
dependencies = [
"displaydoc",
"icu_collections",
"icu_normalizer_data",
"icu_properties",
"icu_provider",
"smallvec 1.13.2",
"utf16_iter",
"utf8_iter",
"write16",
"zerovec",
]
[[package]]
name = "icu_normalizer_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
[[package]]
name = "icu_properties"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
dependencies = [
"displaydoc",
"icu_collections",
"icu_locid_transform",
"icu_properties_data",
"icu_provider",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_properties_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
[[package]]
name = "icu_provider"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
dependencies = [
"displaydoc",
"icu_locid",
"icu_provider_macros",
"stable_deref_trait",
"tinystr",
"writeable",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_provider_macros"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "idempotency"
version = "0.0.1"
@@ -5607,27 +5461,6 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "idna"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
dependencies = [
"idna_adapter",
"smallvec 1.13.2",
"utf8_iter",
]
[[package]]
name = "idna_adapter"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
dependencies = [
"icu_normalizer",
"icu_properties",
]
[[package]]
name = "image"
version = "0.25.6"
@@ -5813,7 +5646,7 @@ dependencies = [
"rand 0.8.5",
"redis_common",
"redis_schema",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_derive",
"serde_json",
@@ -6047,11 +5880,10 @@ checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e"
[[package]]
name = "js-sys"
version = "0.3.77"
version = "0.3.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
dependencies = [
"once_cell",
"wasm-bindgen",
]
@@ -6226,7 +6058,7 @@ dependencies = [
"futures-util",
"hostname 0.3.1",
"httpdate",
"idna 0.5.0",
"idna",
"mime",
"native-tls",
"nom",
@@ -6446,18 +6278,6 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9"
[[package]]
name = "litemap"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
[[package]]
name = "litrs"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
[[package]]
name = "local-channel"
version = "0.1.5"
@@ -7024,7 +6844,7 @@ dependencies = [
"ort",
"primitives",
"rand 0.8.5",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_json",
"serde_yaml",
@@ -7201,7 +7021,7 @@ dependencies = [
"regex",
"regex-automata",
"regex-syntax",
"reqwest 0.12.15",
"reqwest 0.12.12",
"rsa",
"rustls-pki-types",
"semver 1.0.23",
@@ -7987,25 +7807,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openai_sora_client"
version = "0.0.1"
dependencies = [
"anyhow",
"crockford",
"errors",
"filesys",
"log",
"my-workspace-hack",
"reqwest 0.12.15",
"serde",
"serde_derive",
"serde_json",
"subprocess_common",
"testing",
"tokio",
]
[[package]]
name = "openssl"
version = "0.10.71"
@@ -8924,12 +8725,6 @@ dependencies = [
"prost",
]
[[package]]
name = "psl-types"
version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
[[package]]
name = "psm"
version = "0.1.21"
@@ -8959,16 +8754,6 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "publicsuffix"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
dependencies = [
"idna 1.0.3",
"psl-types",
]
[[package]]
name = "pulldown-cmark"
version = "0.9.3"
@@ -9698,14 +9483,12 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.12.15"
version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
dependencies = [
"base64 0.22.1",
"bytes 1.10.0",
"cookie 0.18.1",
"cookie_store",
"encoding_rs",
"futures-channel",
"futures-core",
@@ -9722,7 +9505,6 @@ dependencies = [
"js-sys",
"log",
"mime",
"mime_guess",
"native-tls",
"once_cell",
"percent-encoding",
@@ -9935,7 +9717,7 @@ dependencies = [
"minidom",
"percent-encoding",
"quick-xml 0.36.2",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_derive",
"serde_json",
@@ -10389,9 +10171,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
version = "1.0.219"
version = "1.0.210"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
dependencies = [
"serde_derive",
]
@@ -10409,9 +10191,9 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.219"
version = "1.0.210"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
dependencies = [
"proc-macro2",
"quote",
@@ -10444,9 +10226,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.140"
version = "1.0.128"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
dependencies = [
"indexmap 2.2.6",
"itoa 1.0.11",
@@ -11294,7 +11076,6 @@ dependencies = [
"mysql_queries",
"native-tls",
"once_cell",
"openai_sora_client",
"password",
"pin-project-lite",
"primitives",
@@ -11306,7 +11087,7 @@ dependencies = [
"redis_common",
"redis_schema",
"regex",
"reqwest 0.12.15",
"reqwest 0.12.12",
"reusable_types",
"rust-s3",
"serde",
@@ -12070,7 +11851,7 @@ dependencies = [
"percent-encoding",
"plist",
"raw-window-handle",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_json",
"serde_repr",
@@ -12429,7 +12210,7 @@ dependencies = [
"easyenv",
"log",
"my-workspace-hack",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde_json",
]
@@ -12497,16 +12278,6 @@ dependencies = [
"crunchy",
]
[[package]]
name = "tinystr"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
dependencies = [
"displaydoc",
"zerovec",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
@@ -13128,7 +12899,7 @@ dependencies = [
"my-workspace-hack",
"mysql_queries",
"r2d2_redis",
"reqwest 0.12.15",
"reqwest 0.12.12",
"serde",
"serde_derive",
"serde_json",
@@ -13389,7 +13160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
dependencies = [
"form_urlencoded",
"idna 0.5.0",
"idna",
"percent-encoding",
"serde",
]
@@ -13448,24 +13219,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "utf16_iter"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8-width"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
[[package]]
name = "utf8_iter"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -13506,7 +13265,7 @@ dependencies = [
"actix-web",
"mime_guess",
"regex",
"reqwest 0.12.15",
"reqwest 0.12.12",
"rust-embed",
"serde",
"serde_json",
@@ -13743,24 +13502,23 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
dependencies = [
"cfg-if 1.0.0",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.100"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.87",
@@ -13781,9 +13539,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.100"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -13791,9 +13549,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.100"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
@@ -13804,12 +13562,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.100"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
dependencies = [
"unicode-ident",
]
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
[[package]]
name = "wasm-streams"
@@ -13855,7 +13610,7 @@ dependencies = [
"log",
"my-workspace-hack",
"once_cell",
"reqwest 0.12.15",
"reqwest 0.12.12",
"rss",
"scraper",
"serde",
@@ -14109,7 +13864,7 @@ dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.2.0",
"windows-strings 0.1.0",
"windows-strings",
"windows-targets 0.52.6",
]
@@ -14143,13 +13898,13 @@ checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
[[package]]
name = "windows-registry"
version = "0.4.0"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result 0.3.2",
"windows-strings 0.3.1",
"windows-targets 0.53.0",
"windows-result 0.2.0",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
@@ -14170,15 +13925,6 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
@@ -14189,15 +13935,6 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
@@ -14532,18 +14269,6 @@ dependencies = [
"bitflags 2.5.0",
]
[[package]]
name = "write16"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
[[package]]
name = "writeable"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
[[package]]
name = "wry"
version = "0.48.1"
@@ -14723,28 +14448,6 @@ dependencies = [
"syn 2.0.87",
]
[[package]]
name = "zerovec"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
dependencies = [
"yoke",
"zerofrom",
"zerovec-derive",
]
[[package]]
name = "zerovec-derive"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "zip"
version = "0.6.6"
-21
View File
@@ -32,7 +32,6 @@ members = [
"crates/lib/api_clients/cloud_storage",
"crates/lib/api_clients/google_drive_common",
"crates/lib/api_clients/google_sign_in",
"crates/lib/api_clients/openai_sora_client",
"crates/lib/api_clients/thumbnail_generator",
"crates/lib/bootstrap",
"crates/lib/bucket_paths",
@@ -119,23 +118,3 @@ members = [
# see usage: https://crates.io/crates/cargo-hakari
"crates/my-workspace-hack",
]
[workspace.package]
publish = false
[workspace.dependencies]
# Hakari workspace hack (faster compile times)
my-workspace-hack = { path = "crates/my-workspace-hack" }
# Internal libs
errors = { path = "crates/lib/errors" }
# External libs
# Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
# If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml
anyhow = "1.0.97"
log = "0.4.27"
reqwest = { version = "0.12.15", features = ["json", "cookies"] }
serde = "1.0.219"
serde_derive = "1.0.219"
serde_json = "1.0.140"
@@ -40,10 +40,6 @@ pub enum InferenceJobType {
#[serde(rename = "studio_gen2")]
StudioGen2,
/// Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
/// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
#[serde(rename = "convert_fbx_gltf")]
ConvertFbxToGltf,
@@ -112,7 +108,6 @@ impl InferenceJobType {
Self::GptSovits => "gpt_sovits",
Self::ComfyUi => "comfy_ui",
Self::StudioGen2 => "studio_gen2",
Self::ImageGenApi => "image_gen_api",
Self::ConvertFbxToGltf => "convert_fbx_gltf",
Self::MocapNet => "mocap_net",
Self::RvcV2 => "rvc_v2",
@@ -137,7 +132,6 @@ impl InferenceJobType {
"gpt_sovits" => Ok(Self::GptSovits),
"comfy_ui" => Ok(Self::ComfyUi),
"studio_gen2" => Ok(Self::StudioGen2),
"image_gen_api" => Ok(Self::ImageGenApi),
"convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
"mocap_net" => Ok(Self::MocapNet),
"rvc_v2" => Ok(Self::RvcV2),
@@ -165,7 +159,6 @@ impl InferenceJobType {
Self::GptSovits,
Self::ComfyUi,
Self::StudioGen2,
Self::ImageGenApi,
Self::ConvertFbxToGltf,
Self::MocapNet,
Self::RvcV2,
@@ -204,7 +197,6 @@ mod tests {
assert_serialization(InferenceJobType::GptSovits, "gpt_sovits");
assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
assert_serialization(InferenceJobType::StudioGen2, "studio_gen2");
assert_serialization(InferenceJobType::ImageGenApi, "image_gen_api");
assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
assert_serialization(InferenceJobType::MocapNet, "mocap_net");
assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
@@ -228,7 +220,6 @@ mod tests {
assert_eq!(InferenceJobType::GptSovits.to_str(), "gpt_sovits");
assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
assert_eq!(InferenceJobType::StudioGen2.to_str(), "studio_gen2");
assert_eq!(InferenceJobType::ImageGenApi.to_str(), "image_gen_api");
assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
@@ -252,7 +243,6 @@ mod tests {
assert_eq!(InferenceJobType::from_str("gpt_sovits").unwrap(), InferenceJobType::GptSovits);
assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
assert_eq!(InferenceJobType::from_str("studio_gen2").unwrap(), InferenceJobType::StudioGen2);
assert_eq!(InferenceJobType::from_str("image_gen_api").unwrap(), InferenceJobType::ImageGenApi);
assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
@@ -271,7 +261,7 @@ mod tests {
fn all_variants() {
// Static check
let mut variants = InferenceJobType::all_variants();
assert_eq!(variants.len(), 17);
assert_eq!(variants.len(), 16);
assert_eq!(variants.pop_first(), Some(InferenceJobType::VideoRender));
assert_eq!(variants.pop_first(), Some(InferenceJobType::LivePortrait));
assert_eq!(variants.pop_first(), Some(InferenceJobType::FaceFusion));
@@ -279,7 +269,6 @@ mod tests {
assert_eq!(variants.pop_first(), Some(InferenceJobType::GptSovits));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::StudioGen2));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
@@ -33,10 +33,6 @@ pub enum InferenceModelType {
#[serde(rename = "seed_vc")]
SeedVc,
/// NB: This is for Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
#[serde(rename = "tacotron2")]
Tacotron2,
#[serde(rename = "vits")]
@@ -74,7 +70,6 @@ impl InferenceModelType {
Self::VallEX => "vall_e_x",
Self::RerenderAVideo => "rerender_a_video",
Self::StableDiffusion => "stable_diffusion",
Self::ImageGenApi => "image_gen_api",
Self::SeedVc => "seed_vc",
Self::MocapNet => "mocap_net",
Self::StyleTTS2 => "styletts2",
@@ -95,7 +90,6 @@ impl InferenceModelType {
"vall_e_x" => Ok(Self::VallEX),
"rerender_a_video" => Ok(Self::RerenderAVideo),
"stable_diffusion" => Ok(Self::StableDiffusion),
"image_gen_api" => Ok(Self::ImageGenApi),
"mocap_net" => Ok(Self::MocapNet),
"styletts2" => Ok(Self::StyleTTS2),
"comfy_ui" => Ok(Self::ComfyUi),
@@ -119,7 +113,6 @@ impl InferenceModelType {
Self::VallEX,
Self::RerenderAVideo,
Self::StableDiffusion,
Self::ImageGenApi,
Self::MocapNet,
Self::StyleTTS2,
Self::ConvertFbxToGltf,
@@ -147,7 +140,6 @@ mod tests {
assert_serialization(InferenceModelType::VallEX, "vall_e_x");
assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video");
assert_serialization(InferenceModelType::StableDiffusion, "stable_diffusion");
assert_serialization(InferenceModelType::ImageGenApi, "image_gen_api");
assert_serialization(InferenceModelType::MocapNet, "mocap_net");
assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
@@ -167,7 +159,6 @@ mod tests {
assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x");
assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video");
assert_eq!(InferenceModelType::StableDiffusion.to_str(), "stable_diffusion");
assert_eq!(InferenceModelType::ImageGenApi.to_str(), "image_gen_api");
assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net");
assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
@@ -185,7 +176,6 @@ mod tests {
assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX);
assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo);
assert_eq!(InferenceModelType::from_str("stable_diffusion").unwrap(), InferenceModelType::StableDiffusion);
assert_eq!(InferenceModelType::from_str("image_gen_api").unwrap(), InferenceModelType::ImageGenApi);
assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet);
assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
@@ -197,18 +187,17 @@ mod tests {
fn all_variants() {
// Static check
let mut variants = InferenceModelType::all_variants();
assert_eq!(variants.len(), 15);
assert_eq!(variants.len(), 13);
assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SeedVc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));
assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX));
assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StableDiffusion));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ImageGenApi));
assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet));
assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
@@ -91,10 +91,6 @@ pub enum PublicInferenceJobType {
/// Re-render a video is a video style transfer algorithm. We developed code
/// around it, but chose to develop AnimateDiff / ComfyUI support instead.
RerenderAVideo,
/// Sora GPT 4o image gen
#[serde(rename = "image_gen_api")]
ImageGenApi,
}
/// NB: Legacy API for older code.
@@ -123,7 +119,6 @@ impl PublicInferenceJobType {
InferenceJobType::Unknown => Self::Unknown,
InferenceJobType::BevyToWorkflow => Self::BevyToWorkflow,
InferenceJobType::RerenderAVideo => Self::RerenderAVideo,
InferenceJobType::ImageGenApi => Self::ImageGenApi,
}
}
@@ -145,7 +140,6 @@ impl PublicInferenceJobType {
Self::SeedVc => InferenceJobType::SeedVc,
Self::SoVitsSvc => InferenceJobType::SoVitsSvc,
Self::StableDiffusion => InferenceJobType::StableDiffusion,
Self::ImageGenApi => InferenceJobType::ImageGenApi,
Self::StyleTTS2 => InferenceJobType::StyleTTS2,
Self::Tacotron2 => InferenceJobType::Tacotron2,
Self::Unknown => InferenceJobType::Unknown,
@@ -234,31 +234,17 @@ impl BucketClient {
.map(|s| s.to_string())
.ok_or(anyhow!("could not convert object path to string"))?;
info!("creating file for bucket download: {:?}", filesystem_path.as_ref());
info!("downloading from bucket: {:?}", &object_path_str);
let mut output_file = File::create(filesystem_path).await?;
let mut output_file = tokio::fs::File::create(filesystem_path).await?;
let result = self.bucket.get_object_to_writer(&object_path_str, &mut output_file).await;
info!("downloading from bucket (named '{}'), path: {}", &self.bucket.name, &object_path_str);
let status_code = match result {
Ok(status_code) => status_code,
Err(err) => {
return bail!("Error downloading from bucket (named '{}'): {:?}", &self.bucket.name, err)
}
};
let status_code = self.bucket.get_object_to_writer(&object_path_str, &mut output_file).await?;
match status_code {
404 => {
error!("File not found in bucket (named '{}'), path: {}", &self.bucket.name, &object_path_str);
bail!("File not found in bucket: {}", &object_path_str)
},
_ => {
info!("download code: {}", status_code);
},
404 => bail!("File not found in bucket: {}", &object_path_str),
_ => {},
}
info!("download code: {}", status_code);
Ok(())
}
}
@@ -1,35 +0,0 @@
[package]
name = "openai_sora_client"
edition = "2021"
version = "0.0.1"
publish = false
[package.metadata.cargo-udeps.ignore]
normal = ["my-workspace-hack"]
[lib]
name = "openai_sora_client"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
my-workspace-hack = { workspace = true }
# Internal
crockford = { path = "../../data/crockford" }
errors = { path = "../../errors" }
subprocess_common = { path = "../../subprocess_common" }
# External
anyhow = { workspace = true }
log = { workspace = true }
reqwest = { workspace = true, features = ["multipart"] }
serde = { workspace = true }
serde_derive = { workspace = true }
serde_json = { workspace = true }
tokio = { version = "1.38.0", features = ["macros"] }
[dev-dependencies]
testing = { path = "../../testing" }
filesys = { path = "../../files/filesys" }
@@ -1,43 +0,0 @@
use reqwest::RequestBuilder;
use errors::AnyhowResult;
// NB: It appears that the sentinel may require a matching user agent.
pub (crate) const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
#[derive(Clone)]
pub struct SoraCredentials {
pub bearer_token: String,
pub cookie: String,
pub sentinel: Option<String>,
}
impl SoraCredentials {
pub fn authorization_header_value(&self) -> String {
match self.bearer_token.get(0..6) {
Some("bearer") | Some("Bearer") => self.bearer_token.clone(),
_ => "Bearer ".to_owned() + &self.bearer_token,
}
}
pub fn from_env() -> AnyhowResult<Self> {
let bearer = std::env::var("SORA_BEARER_TOKEN")?;
let cookie = std::env::var("SORA_COOKIE")?;
let sentinel = std::env::var("SORA_SENTINEL").ok();
Ok(Self { bearer_token: bearer, cookie, sentinel })
}
pub fn add_credential_headers_to_request(&self, request: RequestBuilder) -> RequestBuilder {
let bearer_header = self.authorization_header_value();
let mut request = request
.header("User-Agent", USER_AGENT)
.header("Cookie", &self.cookie)
.header("Authorization", bearer_header);
if let Some(sentinel) = &self.sentinel {
request = request.header("OpenAI-Sentinel-Token", sentinel);
}
request
}
}
@@ -1,48 +0,0 @@
#[derive(Debug, Clone)]
pub struct SoraImageGenResponse {
pub task_id: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NumImages {
One,
Two,
Four
}
impl NumImages {
pub fn as_count(&self) -> usize {
match self {
NumImages::One => 1,
NumImages::Two => 2,
NumImages::Four => 4,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ImageSize {
Square,
Wide,
Tall,
}
impl ImageSize {
// TODO: Verify these dimensions. 480x480 is correct.
pub fn as_dimensions(&self) -> (u16, u16) {
match self {
ImageSize::Square => (480, 480),
ImageSize::Wide => (854, 480),
ImageSize::Tall => (480, 854),
}
}
pub fn as_width(&self) -> u16 {
self.as_dimensions().0
}
pub fn as_height(&self) -> u16 {
self.as_dimensions().1
}
}
@@ -1,130 +0,0 @@
use anyhow::anyhow;
use serde_derive::{Deserialize, Serialize};
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
/// This user agent is tied to the sentinel generation. If we need to change it, we may need to change sentinel generation too.
const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) enum VideoGenType {
ImageGen,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) enum OperationType {
/// Simple prompt without reference images
SimpleCompose,
/// Prompt with reference images
Remix,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) enum InpaintItemType {
Image,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct InpaintItem {
pub r#type: InpaintItemType,
/// Typically "0"
pub frame_index: u32,
/// Unknown field; defaults to null
pub preset_id: Option<String>,
/// Unknown field; defaults to null
pub generation_id: Option<String>,
/// Token identifier of the item, eg. "media_01jqt9vt20erx9zvryf3v1pecx"
pub upload_media_id: String,
/// Typically "0"
pub source_start_frame: u32,
/// Typically "0"
pub source_end_frame: u32,
/// Unknown field; defaults to null
pub crop_bounds: Option<String>,
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct RawSoraImageGenRequest {
/// eg. "image_gen"
pub r#type: VideoGenType,
/// eg. "simple_compose"
pub operation: OperationType,
/// The user's raw prompt
pub prompt: String,
/// The number of variants to generate, eg. 2
pub n_variants: usize,
/// The width of the image, eg. 480
pub width: u16,
/// The height of the image, eg. 480
pub height: u16,
/// The number of frames to generate, eg. 1
pub n_frames: u8,
/// The items to inpaint, eg. []
pub inpaint_items: Vec<InpaintItem>,
}
#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct RawSoraResponse {
/// eg. "task_01jqsz9dsae9tvjygf1abrv3xf"
pub id: String,
/// not known
pub priority: Option<String>,
}
// https://sora.com/backend/notif?limit=100&before=task_01jqpg8qghenet0rw2a79p0vbn
// error
// message "Missing bearer authentication in header"
// type "invalid_request_error"
// param null
// code null
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub (crate) struct RawSoraErrorResponse {
error: String,
message: String,
r#type: String,
param: Option<String>,
code: Option<String>,
}
/// Don't expose the internal request implementation as there are only a few "correct" ways to call the API.
pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: &SoraCredentials) -> AnyhowResult<RawSoraResponse> {
let client = reqwest::Client::new();
let sentinel = match credentials.sentinel.as_deref() {
None => return Err(anyhow!("Calls to image gen require a sentinel in the credentials payload.")),
Some(sentinel) => sentinel,
};
let http_request = client.post(SORA_IMAGE_GEN_URL)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie)
.header("Authorization", credentials.authorization_header_value())
.header("Content-Type", "application/json")
.header("OpenAI-Sentinel-Token", sentinel);
let http_request = http_request.json(&sora_request).build()?;
let response = client.execute(http_request)
.await?;
let status = response.status();
let response_body = &response.text().await?;
if status != reqwest::StatusCode::OK {
return Err(anyhow::anyhow!("the request failed; status = {:?}, message = {:?}", status, response_body));
}
let response = serde_json::from_str(response_body)?;
Ok(response)
}
@@ -1,4 +0,0 @@
pub (crate) mod image_gen_http_request;
pub mod common;
pub mod sora_image_gen_remix;
pub mod sora_image_gen_simple;
@@ -1,87 +0,0 @@
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, VideoGenType};
pub struct SoraImageGenRemixRequest<'a> {
pub prompt: String,
pub num_images: NumImages,
pub image_size: ImageSize,
pub sora_media_tokens: Vec<String>,
pub credentials: &'a SoraCredentials,
}
/// The "remix" commands let you supply additional images as context.
/// Sora "media tokens" of previously uploaded images must be supplied.
pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
let args = RawSoraImageGenRequest {
r#type: VideoGenType::ImageGen,
operation: OperationType::Remix,
prompt: request.prompt,
n_variants: request.num_images.as_count(),
width: request.image_size.as_width(),
height: request.image_size.as_height(),
n_frames: 1,
inpaint_items: request.sora_media_tokens.into_iter().map(|token| {
InpaintItem {
r#type: InpaintItemType::Image,
frame_index: 0,
preset_id: None,
generation_id: None,
upload_media_id: token,
source_start_frame: 0,
source_end_frame: 0,
crop_bounds: None,
}
}).collect(),
};
// TODO: Error handling.
let result = image_gen_http_request(args, &request.credentials).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
})
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages};
use crate::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let sentinel = read_to_string(test_file_path("test_data/temp/sentinel.txt")?)?;
let sentinel = sentinel.trim().to_string();
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
let response = sora_image_gen_remix(SoraImageGenRemixRequest {
prompt: "Match the pose and scene layout from the uploaded image exactly. A smart dog getting kisses from another dog. Brick building. Anime style".to_string(),
num_images: NumImages::One,
image_size: ImageSize::Square,
sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()],
credentials: &creds,
}).await?;
println!("task_id: {}", response.task_id);
assert!(response.task_id.starts_with("task_"));
Ok(())
}
}
@@ -1,72 +0,0 @@
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
use crate::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType};
pub struct SoraImageGenSimpleRequest<'a> {
pub prompt: String,
pub num_images: NumImages,
pub image_size: ImageSize,
pub credentials: &'a SoraCredentials,
}
pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
let sora_request = RawSoraImageGenRequest {
r#type: VideoGenType::ImageGen,
operation: OperationType::SimpleCompose,
prompt: args.prompt,
n_variants: args.num_images.as_count(),
width: args.image_size.as_width(),
height: args.image_size.as_height(),
n_frames: 1,
inpaint_items: vec![],
};
// TODO: Error handling.
let result = image_gen_http_request(sora_request, args.credentials).await?;
Ok(SoraImageGenResponse {
task_id: result.id,
})
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::image_gen::common::{ImageSize, NumImages};
use crate::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest};
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let sentinel = read_to_string(test_file_path("test_data/temp/sentinel.txt")?)?;
let sentinel = sentinel.trim().to_string();
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
let response = sora_image_gen_simple(SoraImageGenSimpleRequest {
prompt: "A pirate and a ninja fight in a battle inside a UFO. Fully photo realistic, lifelike, lens flare".to_string(),
num_images: NumImages::One,
image_size: ImageSize::Square,
credentials: &creds,
}).await?;
println!("task_id: {}", response.task_id);
assert!(response.task_id.starts_with("task_"));
Ok(())
}
}
@@ -1 +0,0 @@
pub mod sora_job_status;
@@ -1,64 +0,0 @@
const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
// https://sora.com/backend/notif?limit=100&before=task_01jqwwrkvgeqp8jsf5mqk1jceh
use crate::credentials::SoraCredentials;
pub async fn sora_job_status(
task_id: &str,
credentials: &SoraCredentials,
) -> anyhow::Result<()> {
let url = format!("https://sora.com/backend/notif?limit=100&before={}", task_id);
let client = reqwest::Client::new();
let request = client
.get(url)
.header("Authorization", &credentials.bearer_token)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie);
let response = request.send().await?;
let json_response = &response.text().await?;
println!(" >>> response = {:?}", json_response);
//let response = serde_json::from_str(json_response)?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::job::sora_job_status::sora_job_status;
#[ignore]
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let sentinel = read_to_string(test_file_path("test_data/temp/sentinel.txt")?)?;
let sentinel = sentinel.trim().to_string();
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: Some(sentinel),
};
let task_id = "task_01jqwwrkvgeqp8jsf5mqk1jceh";
let result = sora_job_status(task_id, &creds).await?;
Ok(())
}
}
@@ -1,4 +0,0 @@
pub mod credentials;
pub mod image_gen;
pub mod job;
pub mod upload;
@@ -1,3 +0,0 @@
pub (crate) mod upload_media_http_request;
pub mod upload_media_from_bytes;
pub mod upload_media_from_file;
@@ -1,29 +0,0 @@
use std::path::{Path, PathBuf};
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
pub struct SoraMediaUploadRequest<'a> {
pub file_path: String,
pub credentials: &'a SoraCredentials,
}
/// Upload bytes.
/// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference.
pub async fn sora_media_upload_from_bytes<P: AsRef<Path>>(bytes: Vec<u8>, file_name: String, creds: &SoraCredentials) -> AnyhowResult<SoraMediaUploadResponse> {
let file_path = PathBuf::from(&file_name);
// TODO: Read file magic bytes first, then fall back to this.
let mime_type = match file_path.extension().and_then(|e| e.to_str()) {
Some("jpg") | Some("jpeg") => "image/jpeg",
Some("png") => "image/png",
// Some("webp") => "image/webp",
// Some("gif") => "image/gif",
// Some("mp4") => "video/mp4",
// Some("mov") => "video/quicktime",
// Some("webm") => "video/webm",
_ => "application/octet-stream",
};
Ok(upload_media_http_request(bytes, file_name, mime_type, creds).await?)
}
@@ -1,81 +0,0 @@
use std::path::Path;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use errors::AnyhowResult;
use crate::credentials::SoraCredentials;
use crate::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
/// Try to prevent buffer reallocations.
/// There's a better way to implement this.
const INITIAL_BUFFER_SIZE : usize = 1024*1024;
pub struct SoraMediaUploadRequest<'a> {
pub file_path: String,
pub credentials: &'a SoraCredentials,
}
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: &SoraCredentials) -> AnyhowResult<SoraMediaUploadResponse> {
let mut file = File::open(&file_path).await?;
let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE);
file.read_to_end(&mut buffer).await?;
let filename = file_path.as_ref().file_name()
.ok_or_else(|| anyhow::anyhow!("Invalid file path"))?
.to_string_lossy()
.to_string();
// TODO: Read file magic bytes first, then fall back to this.
let mime_type = match file_path.as_ref().extension().and_then(|e| e.to_str()) {
Some("jpg") | Some("jpeg") => "image/jpeg",
Some("png") => "image/png",
// Some("webp") => "image/webp",
// Some("gif") => "image/gif",
// Some("mp4") => "video/mp4",
// Some("mov") => "video/quicktime",
// Some("webm") => "video/webm",
_ => "application/octet-stream",
};
Ok(upload_media_http_request(buffer, filename, mime_type, creds).await?)
}
#[cfg(test)]
mod tests {
use std::fs::read_to_string;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::credentials::SoraCredentials;
use crate::upload::upload_media_from_file::sora_media_upload_from_file;
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
#[tokio::test]
pub async fn manual_test() -> AnyhowResult<()> {
let cookie = read_to_string(test_file_path("test_data/temp/cookie.txt")?)?;
let cookie = cookie.trim().to_string();
let bearer = read_to_string(test_file_path("test_data/temp/bearer.txt")?)?;
let bearer = bearer.trim().to_string();
let image_path = test_file_path("test_data/image/juno.jpg")?; // media_01jqyqgqpwf40tkcapq5bmaz5d
let creds = SoraCredentials {
bearer_token: bearer,
cookie,
sentinel: None,
};
let response = sora_media_upload_from_file(
image_path,
&creds,
).await?;
println!("media: {:?}", response);
println!("media_id: {}", response.id);
println!("media_url: {}", response.url);
assert!(response.id.starts_with("media_"));
Ok(())
}
}
@@ -1,81 +0,0 @@
use crate::credentials::{SoraCredentials, USER_AGENT};
use errors::AnyhowResult;
use reqwest::multipart::{Form, Part};
use reqwest::Client;
use serde::Deserialize;
const SORA_UPLOAD_MEDIA_URL: &str = "https://sora.com/backend/uploads";
// Content-Disposition: form-data; name="file"; filename="imagename.webp"
/*
Response:
{
"id": "media_01jqt9vt20erx9zvryf3v1pecx",
"type": "image",
"created_at": "2025-04-02T03:49:04.185867Z",
"filename": "Rith_essa.webp",
"extension": "jpeg",
"mime_type": "image/jpeg",
"url": "https://videos.openai.com/vg-assets/assets%2Fclient_upload%2Fmedia%2Fuser-9wf6JFdgRSJLjvSJ53LNAbV8%2Fmedia_01jqt9vt20erx9zvryf3v1pecx.jpeg?st=2025-04-02T02%3A39%3A01Z&se=2025-04-08T03%3A39%3A01Z&sks=b&skt=2025-04-02T02%3A39%3A01Z&ske=2025-04-08T03%3A39%3A01Z&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skoid=3d249c53-07fa-4ba4-9b65-0bf8eb4ea46a&skv=2019-02-02&sv=2018-11-09&sr=b&sp=r&spr=https%2Chttp&sig=%2BdnTQyw9OFF2kaVoEReT2cVfTqmKjGt18m7P4wKCWCI%3D&az=oaivgprodscus",
"width": 1200,
"height": 675,
"duration_sec": null,
"n_frames": 1,
"size_bytes": 118501,
"thumbnail_url": "https://videos.openai.com/vg-assets/assets%2Fclient_upload%2Fmedia%2Fuser-9wf6JFdgRSJLjvSJ53LNAbV8%2Fmedia_01jqt9vt20erx9zvryf3v1pecx.jpg?st=2025-04-02T02%3A39%3A01Z&se=2025-04-08T03%3A39%3A01Z&sks=b&skt=2025-04-02T02%3A39%3A01Z&ske=2025-04-08T03%3A39%3A01Z&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skoid=3d249c53-07fa-4ba4-9b65-0bf8eb4ea46a&skv=2019-02-02&sv=2018-11-09&sr=b&sp=r&spr=https%2Chttp&sig=9aU9xKI0BkhRtrTb0RD05%2BT6prBnmXk1I7nV2nTsFbY%3D&az=oaivgprodscus"
}
*/
#[derive(Debug, Deserialize)]
pub struct SoraMediaUploadResponse {
/// This is the media token id.
pub id: String,
pub r#type: String,
pub created_at: String,
pub filename: String,
pub extension: String,
pub mime_type: String,
pub url: String,
pub width: Option<u32>,
pub height: Option<u32>,
pub duration_sec: Option<f64>,
pub n_frames: Option<u32>,
pub size_bytes: u64,
pub thumbnail_url: Option<String>,
}
pub (crate) struct SoraMediaUploadRequest<'a> {
pub file_path: String,
pub credentials: &'a SoraCredentials,
}
pub (crate) async fn upload_media_http_request(file_bytes: Vec<u8>, filename: String, mime_type: &str, credentials: &SoraCredentials) -> AnyhowResult<SoraMediaUploadResponse> {
// Create multipart form
let part = Part::bytes(file_bytes) // NB: Reqwest needs to own the bytes.
.file_name(filename) // NB: Reqwest needs to own the bytes
.mime_str(mime_type)?;
let form = Form::new().part("file", part);
// Make API request
let client = Client::new();
let request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
.multipart(form)
.header("User-Agent", USER_AGENT)
.header("Cookie", &credentials.cookie)
.header("Authorization", credentials.authorization_header_value());
let response = request_builder.send().await?;
// Check response status
if !response.status().is_success() {
let status = response.status();
let text = response.text().await?;
return Err(anyhow::anyhow!("Upload failed with status {}: {}", status, text));
}
// Parse response
let upload_response = response.json::<SoraMediaUploadResponse>().await?;
Ok(upload_response)
}
@@ -1,6 +1,7 @@
use crate::payloads::generic_inference_args::inner_payloads::f5_tts_payload::F5TTSPayload;
use crate::payloads::generic_inference_args::inner_payloads::face_fusion_payload::FaceFusionPayload;
use crate::payloads::generic_inference_args::inner_payloads::gptsovits_payload::GptSovitsPayload;
use crate::payloads::generic_inference_args::inner_payloads::image_generation_payload::StableDiffusionArgs;
use crate::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncArgs;
use crate::payloads::generic_inference_args::inner_payloads::live_portrait_payload::LivePortraitPayload;
use crate::payloads::generic_inference_args::inner_payloads::mocap_payload::MocapArgs;
@@ -11,8 +12,6 @@ use crate::payloads::generic_inference_args::inner_payloads::tts_payload::TTSArg
use crate::payloads::generic_inference_args::inner_payloads::videofilter_payload::RerenderArgs;
use crate::payloads::generic_inference_args::inner_payloads::workflow_payload::WorkflowArgs;
use errors::AnyhowResult;
use crate::payloads::generic_inference_args::inner_payloads::image_generation_payload::StableDiffusionArgs;
use crate::payloads::generic_inference_args::inner_payloads::sora_image_gen_args::SoraImageGenArgs;
/// Used to encode extra state for the `generic_inference_jobs` table.
/// This should act somewhat like a serialized protobuf stored inside a record.
@@ -112,7 +111,7 @@ pub enum PolymorphicInferenceArgs {
Tts(TTSArgs),
// No arguments yet.
// It might be best to just not include this when not used.
/// Voice conversion. (Short name to save space when serializing.)
Vc {
/// Argument for so-vits-svc
@@ -144,10 +143,6 @@ pub enum PolymorphicInferenceArgs {
/// Image generation. (Short name to save space when serializing.)
Ig(StableDiffusionArgs),
/// Sora Image Generation. (Short name to save space when serializing.)
Sg(SoraImageGenArgs),
/// Mocap (Short name to save space when serializing.)
Mc(MocapArgs),
@@ -1,4 +1,3 @@
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
/// Video sources can be one of several:
@@ -53,7 +52,7 @@ pub struct StableDiffusionArgs {
#[serde(rename = "se")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_seed: Option<i64>,
#[serde(rename = "mu")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_upload_path: Option<String>,
@@ -61,7 +60,7 @@ pub struct StableDiffusionArgs {
#[serde(rename = "cf")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_cfg_scale: Option<u32>,
#[serde(rename = "lu")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_lora_upload_path: Option<String>,
@@ -1,4 +1,3 @@
pub mod f5_tts_payload;
pub mod face_fusion_payload;
pub mod gptsovits_payload;
pub mod image_generation_payload;
@@ -6,9 +5,9 @@ pub mod lipsync_payload;
pub mod live_portrait_payload;
pub mod mocap_payload;
pub mod render_engine_scene_to_video_payload;
pub mod seed_vc_payload;
pub mod sora_image_gen_args;
pub mod studio_gen2_payload;
pub mod tts_payload;
pub mod videofilter_payload;
pub mod workflow_payload;
pub mod f5_tts_payload;
pub mod seed_vc_payload;
@@ -1,23 +0,0 @@
use tokens::tokens::media_files::MediaFileToken;
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct SoraImageGenArgs {
#[serde(rename = "p")]
#[serde(skip_serializing_if = "Option::is_none")]
pub prompt: Option<String>,
#[serde(rename = "sm")]
pub scene_snapshot_media_token: Option<MediaFileToken>,
#[serde(rename = "mf")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_additional_media_file_tokens: Option<Vec<MediaFileToken>>,
#[serde(rename = "sa")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_number_of_samples: Option<u32>,
#[serde(rename = "su")]
#[serde(skip_serializing_if = "Option::is_none")]
pub maybe_sora_media_upload_tokens: Option<Vec<String>>,
}
-2
View File
@@ -17,9 +17,7 @@ pub fn test_file_path<P: AsRef<Path>>(path_from_repo_root: P) -> AnyhowResult<Pa
.to_str()
.ok_or(anyhow!("path cannot convert to string"))?;
path.push(format!("../../../{}", file));
println!(" >>> path = {:?}", path);
Ok(path.canonicalize()?)
}
#[cfg(test)]
@@ -69,7 +69,6 @@ tts_common = { path = "../../../lib/tts_common" }
url_config = { path = "../../../lib/url_config" }
user_input_common = { path = "../../../lib/user_input_common" }
videos = { path = "../../../lib/files/videos" }
openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" }
# Internal Components
billing_component = { path = "../../plugins/billing" }
@@ -124,8 +124,6 @@ use crate::http_server::endpoints::stats::get_unified_queue_stats_handler::*;
use crate::http_server::endpoints::tags::list_tags_for_entity_handler::*;
use crate::http_server::endpoints::tags::set_tags_for_entity_handler::*;
use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::*;
use crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::*;
use crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::*;
use crate::http_server::endpoints::user_bookmarks::batch_get_user_bookmarks_handler::*;
use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::*;
use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::*;
@@ -209,8 +207,6 @@ use crate::http_server::web_utils::response_success_helpers::*;
crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_image_media_file_handler::upload_image_media_file_handler,
crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::enqueue_studio_image_generation_handler,
crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::upload_snapshot_media_file_handler,
crate::http_server::endpoints::studio_gen2::enqueue_studio_gen2_handler::enqueue_studio_gen2_handler,
crate::http_server::endpoints::media_files::upload::upload_new_engine_asset_media_file_handler::upload_new_engine_asset_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::upload_new_scene_media_file_handler,
@@ -340,9 +336,6 @@ use crate::http_server::web_utils::response_success_helpers::*;
AppStateServerInfo,
AppStateStatusAlertCategory,
AppStateStatusAlertInfo,
EnqueueStudioImageGenRequest,
EnqueueImageGenRequestSuccessResponse,
EnqueueImageGenRequestError,
AppStateSubscriptionProductKey,
AppStateUserInfo,
AppStateUserLocale,
@@ -354,8 +347,6 @@ use crate::http_server::web_utils::response_success_helpers::*;
BatchGetMediaFilesQueryParams,
BatchGetMediaFilesSuccessResponse,
BatchGetUserBookmarksError,
UploadSnapshotMediaFileForm,
UploadSnapshotMediaFileSuccessResponse,
BatchGetUserBookmarksQueryParams,
BatchGetUserBookmarksResponse,
BatchGetUserRatingError,
@@ -1 +0,0 @@
pub mod set_sora_secret_handler;
@@ -1,137 +0,0 @@
use std::fmt;
use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web, Either};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Form, Json, Path};
use anyhow::anyhow;
use log::{error, log, warn};
use r2d2_redis::redis::Commands;
use utoipa::ToSchema;
use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use mysql_queries::queries::media_files::edit::update_media_file_animation_type::update_media_file_animation_type;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use tokens::tokens::media_files::MediaFileToken;
use crate::state::server_state::ServerState;
use crate::util::redis_sora_secrets::get_sora_credentials;
use crate::util::redis_sora_secrets::set_sora_credential_field;
use crate::util::redis_sora_secrets::RedisSoraCredentialSubkey;
#[derive(Deserialize, Clone, Copy)]
#[serde(rename_all = "snake_case")]
pub enum SetSoraSecretType {
Bearer,
Cookie,
Sentinel
}
#[derive(Deserialize, ToSchema)]
pub struct SetSoraSecretRequest {
pub key: SetSoraSecretType,
pub value: String,
}
#[derive(Serialize, ToSchema)]
pub struct SetSoraSecretSuccessResponse {
pub success: bool,
pub bearer: String,
pub cookie: String,
pub sentinel: String,
}
// =============== Error Response ===============
#[derive(Debug, Serialize, ToSchema)]
pub enum SetSoraSecretError {
BadInput(String),
NotAuthorized,
ServerError,
}
impl ResponseError for SetSoraSecretError {
fn status_code(&self) -> StatusCode {
match *self {
SetSoraSecretError::BadInput(_) => StatusCode::BAD_REQUEST,
SetSoraSecretError::NotAuthorized => StatusCode::UNAUTHORIZED,
SetSoraSecretError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for SetSoraSecretError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// =============== Handler ===============
/// Set a Sora secret
#[utoipa::path(
post,
tag = "Control Plane",
path = "/v1/control_plane/sora_secret",
responses(
(status = 200, description = "Success", body = SetSoraSecretSuccessResponse),
(status = 400, description = "Bad input", body = SetSoraSecretError),
(status = 401, description = "Not authorized", body = SetSoraSecretError),
(status = 500, description = "Server error", body = SetSoraSecretError),
),
params(
("request" = SetSoraSecretRequest, description = "Payload for Request"),
("path" = MediaFileTokenPathInfo, description = "Path for Request")
)
)]
pub async fn set_sora_secret_handler(
http_request: HttpRequest,
request: Form<SetSoraSecretRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<SetSoraSecretSuccessResponse>, SetSoraSecretError> {
// TODO(bt,2025-04-03): Some form of authentication should be required for this endpoint.
let value = request.value.trim().to_string();
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
set_sora_credential_field(
&mut redis,
match request.key {
SetSoraSecretType::Bearer => RedisSoraCredentialSubkey::Bearer,
SetSoraSecretType::Cookie => RedisSoraCredentialSubkey::Cookie,
SetSoraSecretType::Sentinel => RedisSoraCredentialSubkey::Sentinel,
},
&value
).map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
let credentials = get_sora_credentials(&mut redis)
.map_err(|e| {
error!("redis error: {:?}", e);
SetSoraSecretError::ServerError
})?;
// TODO(bt,2025-04-03): Probably not smart to return, but useful for debugging.
Ok(Json(SetSoraSecretSuccessResponse {
success: true,
bearer: credentials.bearer_token,
cookie: credentials.cookie,
sentinel: credentials.sentinel.unwrap_or_else(|| "".to_string()),
}))
}
@@ -507,9 +507,6 @@ fn validate_request(
Ok(())
}
fn inference_mode_from_http_request(http_request: &HttpRequest) -> Option<TypeOfInference> {
inference_mode_from_url_path(http_request.path())
}
@@ -1,68 +0,0 @@
use actix_web::HttpRequest;
use anyhow::anyhow;
use log::{error, info};
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::RedisConnectionManager;
use errors::AnyhowResult;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use openai_sora_client::credentials::SoraCredentials;
use crate::util::redis_sora_secrets::get_sora_credentials;
const SORA_BEARER_HEADER_NAME : &str = "sora-bearer";
const SORA_COOKIE_HEADER_NAME : &str = "sora-cookie";
const SORA_SENTINEL_HEADER_NAME : &str = "sora-sentinel";
pub fn get_sora_credentials_from_request(
http_request: &HttpRequest,
redis: &mut PooledConnection<RedisConnectionManager>,
) -> AnyhowResult<SoraCredentials> {
let sora_bearer = get_request_header_optional(&http_request, SORA_BEARER_HEADER_NAME);
let sora_cookie = get_request_header_optional(&http_request, SORA_COOKIE_HEADER_NAME);
let sora_sentinel = get_request_header_optional(&http_request, SORA_SENTINEL_HEADER_NAME);
match (&sora_bearer, &sora_cookie, &sora_sentinel) {
(Some(bearer), Some(cookie), Some(sentinel)) => {
info!("Sora credentials were present in HTTP request headers!");
return Ok(SoraCredentials {
bearer_token: bearer.to_string(),
cookie: cookie.to_string(),
sentinel: Some(sentinel.to_string()),
});
}
_ => {} // Fall through
}
let redis_result = get_sora_credentials(redis);
match redis_result {
Ok(mut credentials) => {
if let Some(bearer) = sora_bearer {
info!("override bearer value");
credentials.bearer_token = bearer;
}
if let Some(cookie) = sora_cookie {
info!("override cookie value");
credentials.cookie = cookie;
}
if let Some(sentinel) = sora_sentinel {
info!("override sentinel value");
credentials.sentinel = Some(sentinel);
}
return Ok(credentials)
},
Err(err) => {
error!("Failed to get Sora credentials from Redis; using env as backup. Error: {:?}", err);
// NB: Fall through
}
};
match SoraCredentials::from_env() {
Ok(creds) => Ok(creds),
Err(err) => {
error!("Failed to load Sora credentials from environment: {:?}", err);
Err(anyhow!("Failed to load Sora credentials from environment"))
}
}
}
@@ -1,3 +0,0 @@
pub mod upload;
pub mod prompt;
pub mod get_sora_credentials_from_request;
@@ -1,465 +0,0 @@
#![forbid(unused_mut)]
use std::fmt::Debug;
use std::fmt::{Display, Formatter};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use log::{debug, error, info};
use log::warn;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use openai_sora_client::credentials::SoraCredentials;
use serde::Deserialize;
use serde::Serialize;
use sqlx::MySqlPool;
use tempdir::TempDir;
use utoipa::ToSchema;
use web::Data;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use http_server_common::request::get_request_ip::get_request_ip;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::{
GenericInferenceArgs,
InferenceCategoryAbbreviated,
PolymorphicInferenceArgs,
};
use mysql_queries::payloads::generic_inference_args::inner_payloads::sora_image_gen_args::SoraImageGenArgs;
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{
insert_generic_inference_job,
InsertGenericInferenceArgs,
};
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use openai_sora_client::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::upload::upload_media_from_file::{sora_media_upload_from_file, SoraMediaUploadRequest};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
use crate::http_server::endpoints::image_studio::get_sora_credentials_from_request::get_sora_credentials_from_request;
use crate::http_server::endpoints::tts::get_tts_inference_job_status::GetTtsInferenceStatusError;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::state::server_state::ServerState;
use crate::util::allowed_studio_access::allowed_studio_access;
use crate::util::redis_sora_secrets::get_sora_credentials;
/// This is the number of images (batch size) to generate for each request.
/// We should allow all users to have multiple images generated at once as this
/// is what other providers do.
const MINIMUM_IMAGE_COUNT : u32 = 1;
/// The maximum number of images (batch size) to generate for each request.
const MAXIMUM_IMAGE_COUNT : u32 = 2;
/// Debug requests can get routed to special "debug-only" workers, which can
/// be used to trial new code, run debugging, etc.
const DEBUG_HEADER_NAME: &str = "enable-debug-mode";
/// The routing tag header can send workloads to particular k8s hosts.
/// This is useful for catching the live logs or intercepting the job.
const ROUTING_TAG_HEADER_NAME: &str = "routing-tag";
#[derive(Deserialize, ToSchema)]
pub struct EnqueueStudioImageGenRequest {
pub uuid_idempotency_token: String,
/// Image media file; the engine or canvas snapshot (screenshot).
pub snapshot_media_token: MediaFileToken,
/// The user's image generation prompt.
pub prompt: String,
/// Additional images to include (optional). Up to nine images.
pub maybe_additional_images: Option<Vec<MediaFileToken>>,
pub maybe_number_of_samples: Option<u32>,
}
#[derive(Serialize, ToSchema)]
pub struct EnqueueImageGenRequestSuccessResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
#[derive(Debug, ToSchema)]
pub enum EnqueueImageGenRequestError {
BadInput(String),
NotAuthorized,
ServerError,
RateLimited,
}
impl ResponseError for EnqueueImageGenRequestError {
fn status_code(&self) -> StatusCode {
match *self {
EnqueueImageGenRequestError::BadInput(_) => StatusCode::BAD_REQUEST,
EnqueueImageGenRequestError::NotAuthorized => StatusCode::UNAUTHORIZED,
EnqueueImageGenRequestError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
EnqueueImageGenRequestError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
EnqueueImageGenRequestError::BadInput(reason) => reason.to_string(),
EnqueueImageGenRequestError::NotAuthorized => "unauthorized".to_string(),
EnqueueImageGenRequestError::ServerError => "server error".to_string(),
EnqueueImageGenRequestError::RateLimited => "rate limited".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
impl Display for EnqueueImageGenRequestError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
/// Prompt image generation using image studio
#[utoipa::path(
post,
tag = "Image Studio",
path = "/v1/image_studio/prompt",
responses(
(
status = 200,
description = "Enqueue TTS generically",
body = EnqueueImageGenRequestSuccessResponse,
),
(status = 400, description = "Bad input", body = EnqueueImageGenRequestError),
(status = 401, description = "Not authorized", body = EnqueueImageGenRequestError),
(status = 429, description = "Rate limited", body = EnqueueImageGenRequestError),
(status = 500, description = "Server error", body = EnqueueImageGenRequestError)
),
params(("request" = EnqueueStudioImageGenRequest, description = "Payload for Image Generation Request"))
)]
pub async fn enqueue_studio_image_generation_handler(
http_request: HttpRequest,
request: Json<EnqueueStudioImageGenRequest>,
server_state: Data<Arc<ServerState>>
) -> Result<HttpResponse, EnqueueImageGenRequestError> {
validate_request(&request)?;
let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| {
warn!("MySql pool error: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
// ==================== USER SESSION ==================== //
let maybe_user_session = server_state.session_checker
.maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection).await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
let mut maybe_user_token: Option<UserToken> = None;
if let Some(user_session) = maybe_user_session.as_ref() {
maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token));
}
// ==================== PLANS ==================== //
// Plan should handle "first anonymous use" and "investor" cases.
let plan = get_correct_plan_for_session(
server_state.server_environment_old,
maybe_user_session.as_ref()
);
// Separate priority for animation.
let priority_level = plan.web_vc_base_priority_level();
// ==================== DEBUG MODE + ROUTING TAG ==================== //
let is_debug_request = get_request_header_optional(&http_request, DEBUG_HEADER_NAME).is_some();
let maybe_routing_tag = get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME).map(
|routing_tag| routing_tag.trim().to_string()
);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.role.is_banned {
return Err(EnqueueImageGenRequestError::NotAuthorized);
}
}
// DETECT premium user and queue
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.logged_out,
Some(ref _user) => &server_state.redis_rate_limiters.logged_in,
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
return Err(EnqueueImageGenRequestError::RateLimited);
}
// Get up IP address
let ip_address = get_request_ip(&http_request);
// ==================== HANDLE IDEMPOTENCY ==================== //
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(EnqueueImageGenRequestError::BadInput(reason));
}
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
EnqueueImageGenRequestError::BadInput("invalid idempotency token".to_string())
})?;
// ==================== DOWNLOAD FILES AND UPLOAD TO SORA ==================== //
// TODO: Maybe this moves to a job.
let work_temp_dir = server_state.temp_dir_creator.new_tempdir("image_studio").unwrap();
let public_bucket_client = server_state.public_bucket_client.clone();
let scene_media_token = request.snapshot_media_token.clone();
let scene_media_path = query_and_download_media_file(
&scene_media_token,
&work_temp_dir,
&public_bucket_client,
&server_state.mysql_pool
).await.map_err(|err| {
error!("Failed to download scene media file: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
let mut files_to_upload = Vec::with_capacity(1 + request.maybe_additional_images
.as_ref().map(|v| v.len()).unwrap_or(0));
files_to_upload.push(scene_media_path.clone());
let additional_images = request.maybe_additional_images.clone().unwrap_or_else(|| vec![]);
for media_file_token in &additional_images {
let media_file_path = query_and_download_media_file(
&media_file_token,
&work_temp_dir,
&public_bucket_client,
&server_state.mysql_pool
).await.map_err(|err| {
error!("Failed to download additional media file: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
files_to_upload.push(media_file_path.clone());
}
// ==================== HANDLE SORA CREDENTIALS ==================== //
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
let sora_credentials = get_sora_credentials_from_request(&http_request, &mut redis)
.map_err(|e| {
error!("sora credential error: {:?}", e);
EnqueueImageGenRequestError::ServerError
})?;
// ==================== HANDLE SORA UPLOAD ==================== //
let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len());
for file_path in files_to_upload {
let scene_media_upload_request = SoraMediaUploadRequest {
file_path: scene_media_path.to_string_lossy().to_string(),
credentials: &sora_credentials,
};
let sora_upload_response = sora_media_upload_from_file(file_path, &sora_credentials)
.await
.map_err(|err| {
error!("Failed to upload scene media to Sora: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
debug!("Uploaded media to Sora : {:?}", sora_upload_response);
sora_media_tokens.push(sora_upload_response.id);
}
// ==================== HANDLE SORA PROMPT ==================== //
let number_of_samples = match request.maybe_number_of_samples {
Some(val) => if val > MAXIMUM_IMAGE_COUNT {
MAXIMUM_IMAGE_COUNT
} else if val < MINIMUM_IMAGE_COUNT {
MINIMUM_IMAGE_COUNT
} else {
val
},
None => 1,
};
let prompt = request.prompt.clone();
let response = sora_image_gen_remix(SoraImageGenRemixRequest {
prompt: prompt.clone(),
num_images: NumImages::One,
image_size: ImageSize::Square,
sora_media_tokens: sora_media_tokens.clone(),
credentials: &sora_credentials,
}).await
.map_err(|err| {
error!("Failed to call Sora image generation: {:?}", err);
EnqueueImageGenRequestError::ServerError
})?;
debug!("Sora image gen response: {:?}", response);
// Store the actual inference args that will go to the database
let inference_args = SoraImageGenArgs {
prompt: Some(prompt),
scene_snapshot_media_token: Some(request.snapshot_media_token.clone()),
maybe_additional_media_file_tokens: Some(additional_images),
maybe_number_of_samples: Some(number_of_samples),
maybe_sora_media_upload_tokens: Some(sora_media_tokens),
};
// create the inference args here
let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request);
// create the job record here!
let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
job_type: InferenceJobType::ImageGenApi,
maybe_product_category: None, // This is not a product anymore
inference_category: InferenceCategory::ImageGeneration,
maybe_model_type: Some(InferenceModelType::ImageGenApi), // NB: Model is static during inference
maybe_model_token: None, // NB: Model is static during inference
maybe_input_source_token: None,
maybe_input_source_token_type: None,
maybe_download_url: None,
maybe_cover_image_media_file_token: None,
maybe_raw_inference_text: None,
maybe_max_duration_seconds: None,
maybe_inference_args: Some(GenericInferenceArgs {
inference_category: Some(InferenceCategoryAbbreviated::ImageGeneration),
args: Some(PolymorphicInferenceArgs::Sg(inference_args)),
}),
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
priority_level,
requires_keepalive: false, //reverse ... TODO fix this. we set it base on account is premium or not ...
is_debug_request,
maybe_routing_tag: maybe_routing_tag.as_deref(),
mysql_pool: &server_state.mysql_pool,
}).await;
let job_token = match query_result {
Ok((job_token, _id)) => job_token,
Err(err) => {
warn!("New generic inference job creation DB error: {:?}", err);
if err.had_duplicate_idempotency_token() {
return Err(EnqueueImageGenRequestError::BadInput("Duplicate idempotency token".to_string()));
}
return Err(EnqueueImageGenRequestError::ServerError);
}
};
let response: EnqueueImageGenRequestSuccessResponse = EnqueueImageGenRequestSuccessResponse {
success: true,
inference_job_token: job_token,
};
let body = serde_json::to_string(&response)
.map_err(|_e| EnqueueImageGenRequestError::ServerError)?;
// Error handling 101 rust result type returned like so.
Ok(HttpResponse::Ok().content_type("application/json").body(body))
}
fn validate_request(
request: &Json<EnqueueStudioImageGenRequest>,
) -> Result<(), EnqueueImageGenRequestError> {
Ok(())
}
#[derive(Debug)]
pub enum DownloadMediaFileError {
IoError(std::io::Error),
MediaFileNotFound,
Other(anyhow::Error),
}
impl From<anyhow::Error> for DownloadMediaFileError {
fn from(err: anyhow::Error) -> Self {
DownloadMediaFileError::Other(err)
}
}
async fn query_and_download_media_file(
media_file_token: &MediaFileToken,
download_dir: &TempDir,
bucket_client: &BucketClient,
mysql_pool: &MySqlPool,
) -> Result<PathBuf, DownloadMediaFileError> {
let media_file_result = get_media_file(
&media_file_token,
false,
mysql_pool
).await;
let media_file= match media_file_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media file in database: {:?}", media_file_token);
return Err(DownloadMediaFileError::MediaFileNotFound)
}
Err(e) => {
error!("could not query media file: {:?}", e);
return Err(DownloadMediaFileError::Other(e))
}
};
let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
&media_file.public_bucket_directory_hash,
media_file.maybe_public_bucket_prefix.as_deref(),
media_file.maybe_public_bucket_extension.as_deref());
let extension = media_file.maybe_public_bucket_extension.clone().unwrap_or_else(|| ".jpg".to_string());
let download_filename = format!("download_file_{}.{}", media_file.token.as_str(), extension);
let download_file_path = download_dir.path().join(download_filename);
info!("Downloading from bucket...");
bucket_client.download_file_to_disk(
&media_file_bucket_path.to_full_object_pathbuf(),
&download_file_path,
).await?;
Ok(download_file_path)
}
@@ -1 +0,0 @@
pub mod enqueue_studio_image_generation_handler;
@@ -1 +0,0 @@
pub mod upload_snapshot_media_file_handler;
@@ -1,279 +0,0 @@
use std::collections::HashSet;
use std::io::{BufReader, Bytes, Cursor, Read};
use std::path::PathBuf;
use std::sync::Arc;
use actix_multipart::form::tempfile::TempFile;
use actix_multipart::form::text::Text;
use actix_multipart::form::MultipartForm;
use actix_multipart::Multipart;
use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use log::{error, info, warn};
use once_cell::sync::Lazy;
use utoipa::ToSchema;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mimetypes::mimetype_to_extension::mimetype_to_extension;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult};
/// Form-multipart request fields.
///
/// IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `UploadSnapshotMediaFileForm` (Under "Schema") FOR DETAILS ON FIELDS AND NULLABILITY.
#[derive(MultipartForm, ToSchema)]
#[multipart(duplicate_field = "deny")]
pub struct UploadSnapshotMediaFileForm {
/// UUID for request idempotency
#[multipart(limit = "2 KiB")]
#[schema(value_type = String, format = Binary)]
uuid_idempotency_token: Text<String>,
/// The uploaded "screenshot" / "snapshot" of the engine scene
#[multipart(limit = "512 MiB")]
#[schema(value_type = Vec<u8>, format = Binary)]
snapshot: TempFile,
/// Optional: If an engine scene was used to create this snapshot, provide it here to create a link
#[multipart(limit = "2 KiB")]
#[schema(value_type = Option<MediaFileToken>, format = Binary)]
scene_media_token: Option<Text<MediaFileToken>>,
/// Optional: Visibility of the scene
#[multipart(limit = "2 KiB")]
#[schema(value_type = Option<Visibility>, format = Binary)]
maybe_visibility: Option<Text<Visibility>>,
}
// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*.
#[derive(Serialize, ToSchema)]
pub struct UploadSnapshotMediaFileSuccessResponse {
pub success: bool,
/// The media token of the "screenshot" / "snapshot" we took.
pub snapshot_media_token: MediaFileToken,
}
static ALLOWED_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
HashSet::from([
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
])
});
/// Upload snapshots of the scenes created in the 3D image studio.
#[utoipa::path(
post,
tag = "Image Studio",
path = "/v1/image_studio/scene_snapshot",
responses(
(status = 200, description = "Success Update", body = UploadSnapshotMediaFileSuccessResponse),
(status = 400, description = "Bad input", body = MediaFileUploadError),
(status = 401, description = "Not authorized", body = MediaFileUploadError),
(status = 429, description = "Too many requests", body = MediaFileUploadError),
(status = 500, description = "Server error", body = MediaFileUploadError),
),
params(
(
"request" = UploadSnapshotMediaFileForm,
description = "IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `UploadSnapshotMediaFileForm` (Under 'Schema') FOR DETAILS ON FIELDS AND NULLABILITY."
),
)
)]
pub async fn upload_snapshot_media_file_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
MultipartForm(mut form): MultipartForm<UploadSnapshotMediaFileForm>,
) -> Result<Json<UploadSnapshotMediaFileSuccessResponse>, MediaFileUploadError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await
.map_err(|err| {
error!("MySql pool error: {:?}", err);
MediaFileUploadError::ServerError
})?;
// ==================== READ SESSION ==================== //
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
error!("Session checker error: {:?}", e);
MediaFileUploadError::ServerError
})?;
let maybe_user_token = maybe_user_session
.as_ref()
.map(|session| session.get_strongly_typed_user_token());
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.is_banned {
return Err(MediaFileUploadError::NotAuthorizedVerbose("user is banned".to_string()));
}
}
// ==================== HANDLE IDEMPOTENCY ==================== //
// TODO(bt, 2024-02-26): This should be a transaction.
let uuid_idempotency_token = form.uuid_idempotency_token.as_ref();
if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) {
return Err(MediaFileUploadError::BadInput(reason));
}
insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
MediaFileUploadError::BadInput("invalid idempotency token".to_string())
})?;
// ==================== UPLOAD METADATA ==================== //
let creator_set_visibility = form.maybe_visibility
.map(|visibility| visibility.0)
.or_else(|| {
maybe_user_session
.as_ref()
.map(|user_session| user_session.preferred_tts_result_visibility)
})
.unwrap_or(Visibility::default());
// ==================== USER DATA ==================== //
let ip_address = get_request_ip(&http_request);
// ==================== FILE VALIDATION ==================== //
let mut file_bytes = Vec::new();
form.snapshot.file.read_to_end(&mut file_bytes)
.map_err(|e| {
error!("Problem reading file: {:?}", e);
MediaFileUploadError::ServerError
})?;
let mimetype = get_mimetype_for_bytes(file_bytes.as_ref())
.map(|mimetype| mimetype.to_string())
.ok_or_else(|| {
warn!("Could not determine mimetype for file");
MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string())
})?;
if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) {
// NB: Don't let our error message inject malicious strings
let filtered_mimetype = mimetype
.chars()
.filter(|c| c.is_ascii())
.filter(|c| c.is_alphanumeric() || *c == '/')
.collect::<String>();
return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype)));
}
// ==================== OTHER FILE METADATA ==================== //
let maybe_filename = form.snapshot.file_name.as_deref()
.as_deref()
.map(|filename| PathBuf::from(filename));
let extension = mimetype_to_extension(&mimetype)
.or_else(|| {
maybe_filename
.as_ref()
.and_then(|filename| filename.extension()) // TODO needs dot prefix
.and_then(|ext| ext.to_str())
})
.ok_or_else(|| {
warn!("Could not determine file extension for mimetype: {}", &mimetype);
MediaFileUploadError::ServerError
})?;
let extension = format!(".{extension}"); // NB: needs dot prefix
let file_size_bytes = file_bytes.len();
let hash = sha256_hash_bytes(&file_bytes)
.map_err(|io_error| {
error!("Problem hashing bytes: {:?}", io_error);
MediaFileUploadError::ServerError
})?;
// ==================== UPLOAD AND SAVE ==================== //
const PREFIX : Option<&str> = Some("snapshot_");
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension));
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
server_state.public_bucket_client.upload_file_with_content_type(
public_upload_path.get_full_object_path_str(),
file_bytes.as_ref(),
&mimetype)
.await
.map_err(|e| {
warn!("Upload media bytes to bucket error: {:?}", e);
MediaFileUploadError::ServerError
})?;
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Image),
media_file_type: MediaFileType::Image,
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility,
upload_type: UploadType::Filesystem, // TODO(bt,2024-05-02): This should be a parameter and a well-known enum.
maybe_engine_category: None,
maybe_animation_type: None,
maybe_mime_type: Some(&mimetype),
file_size_bytes: file_size_bytes as u64,
maybe_duration_millis: None,
sha256_checksum: &hash,
maybe_title: Some("Snapshot"),
maybe_scene_source_media_file_token: None,
is_intermediate_system_file: true,
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: Some(&extension),
pool: &server_state.mysql_pool,
})
.await
.map_err(|err| {
warn!("New file creation DB error: {:?}", err);
MediaFileUploadError::ServerError
})?;
info!("new media file id: {} token: {:?}", record_id, &token);
Ok(Json(UploadSnapshotMediaFileSuccessResponse {
success: true,
snapshot_media_token: token,
}))
}
@@ -5,7 +5,6 @@ pub mod app_state;
pub mod beta_keys;
pub mod categories;
pub mod comments;
pub mod control_plane;
pub mod conversion;
pub mod download_job;
pub mod engine;
@@ -13,7 +12,6 @@ pub mod events;
pub mod featured_items;
pub mod flags;
pub mod image_gen;
pub mod image_studio;
pub mod inference_job;
pub mod investor_demo;
pub mod leaderboard;
@@ -32,9 +30,9 @@ pub mod subscriptions;
pub mod tags;
pub mod trending;
pub mod tts;
pub mod users;
pub mod user_bookmarks;
pub mod user_ratings;
pub mod users;
pub mod vocoders;
pub mod voice_clone_requests;
pub mod voice_conversion;
@@ -1,24 +0,0 @@
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
use crate::http_server::endpoints::control_plane::set_sora_secret_handler::set_sora_secret_handler;
pub fn add_control_plane_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/control_plane")
.service(web::resource("/sora_secret")
.route(web::post().to(set_sora_secret_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
}
@@ -1,29 +0,0 @@
use crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::enqueue_studio_image_generation_handler;
use crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::upload_snapshot_media_file_handler;
use actix_http::body::MessageBody;
use actix_service::ServiceFactory;
use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::{web, App, Error, HttpResponse};
pub fn add_image_studio_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
{
app.service(web::scope("/v1/image_studio")
.service(web::resource("/scene_snapshot")
.route(web::post().to(upload_snapshot_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/prompt")
.route(web::post().to(enqueue_studio_image_generation_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
}
@@ -123,8 +123,6 @@ use crate::http_server::endpoints::w2l::get_w2l_template_use_count::get_w2l_temp
use crate::http_server::endpoints::w2l::get_w2l_upload_template_job_status::get_w2l_upload_template_job_status_handler;
use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_handler;
use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_template_mod_approval_handler;
use crate::http_server::routes::add_control_plane_routes::add_control_plane_routes;
use crate::http_server::routes::add_image_studio_routes::add_image_studio_routes;
use crate::http_server::routes::add_studio_gen2_routes::add_studio_gen2_routes;
use crate::http_server::routes::beta_key_routes::add_beta_key_routes;
use crate::http_server::routes::job_routes::add_job_routes;
@@ -173,10 +171,8 @@ pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) ->
app = add_model_download_routes(app);
app = add_workflow_routes(app);
app = add_studio_gen2_routes(app);
app = add_image_studio_routes(app);
app = add_job_routes(app);
app = add_engine_routes(app); /* /v1/engine/... */
app = add_control_plane_routes(app); /* /v1/control_plane/... */
// app = add_image_gen_routes(app);
@@ -8,6 +8,4 @@ pub mod moderation_routes;
pub mod user_routes;
pub mod weights_routes;
pub mod workflow_routes;
mod tag_routes;
mod add_image_studio_routes;
mod add_control_plane_routes;
mod tag_routes;
@@ -11,20 +11,6 @@ pub struct ScopedTempDirCreator {
}
impl ScopedTempDirCreator {
pub fn auto_setup() -> Self {
// Production configuration
let maybe_tmp = easyenv::get_env_pathbuf_optional("TEMP_DIR");
// MacOS
// TODO: Only run this on macOS
let maybe_mac_tmp = easyenv::get_env_pathbuf_optional("TMPDIR");
let directory = maybe_tmp
.or(maybe_mac_tmp)
.unwrap_or_else(|| PathBuf::from("/tmp"));
Self::for_directory(directory)
}
pub fn for_directory<P: AsRef<Path>>(base_dir: P) -> Self {
Self {
@@ -464,7 +464,7 @@ async fn main() -> AnyhowResult<()> {
user_tokens: user_token_troll_bans,
ip_addresses: ip_address_troll_bans,
},
temp_dir_creator: ScopedTempDirCreator::auto_setup(),
temp_dir_creator: ScopedTempDirCreator::for_directory(easyenv::get_env_pathbuf_or_default("TEMP_DIR", "/tmp")),
google_sign_in_cert: GoogleSignInCert::new(),
};
@@ -11,6 +11,5 @@ pub mod enroll_in_studio;
pub mod generate_random_username;
pub mod placeholder_images;
pub mod read_toml_file_to_struct;
pub mod redis_sora_secrets;
pub mod title_to_url_slug;
pub mod troll_user_bans;
@@ -1,67 +0,0 @@
use std::collections::HashMap;
use anyhow::anyhow;
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::redis::Commands;
use r2d2_redis::RedisConnectionManager;
use errors::AnyhowResult;
use openai_sora_client::credentials::SoraCredentials;
/// Redis key
const SORA_SECRET_REDIS_KEY : &str = "sora_secret";
// Fields within the HKEY
const BEARER_SUBKEY : &str = "bearer";
const COOKIE_SUBKEY : &str = "cookie";
const SENTINEL_SUBKEY : &str = "sentinel";
#[derive(Clone,Copy,Debug)]
pub enum RedisSoraCredentialSubkey {
Bearer,
Cookie,
Sentinel,
}
impl RedisSoraCredentialSubkey {
pub fn to_str(&self) -> &'static str {
match self {
RedisSoraCredentialSubkey::Bearer => BEARER_SUBKEY,
RedisSoraCredentialSubkey::Cookie => COOKIE_SUBKEY,
RedisSoraCredentialSubkey::Sentinel => SENTINEL_SUBKEY,
}
}
}
pub fn get_sora_credentials(
redis: &mut PooledConnection<RedisConnectionManager>
) -> AnyhowResult<SoraCredentials> {
let values : HashMap<String, String> = redis.hgetall(SORA_SECRET_REDIS_KEY)
.map_err(|e| anyhow!("Failed to get Sora credentials from Redis: {}", e))?;
let bearer = values.get(BEARER_SUBKEY);
let cookie = values.get(COOKIE_SUBKEY);
let sentinel = values.get(SENTINEL_SUBKEY);
match (bearer, cookie, sentinel) {
(Some(b), Some(c), Some(s)) => {
Ok(SoraCredentials {
bearer_token: b.to_string(),
cookie: c.to_string(),
sentinel: Some(s.to_string()),
})
}
_ => Err(anyhow!("redis sora credential values not present")),
}
}
pub fn set_sora_credential_field(
redis: &mut PooledConnection<RedisConnectionManager>,
subkey: RedisSoraCredentialSubkey,
value: &str,
) -> AnyhowResult<()> {
redis.hset(SORA_SECRET_REDIS_KEY, subkey.to_str(), value)
.map_err(|e| anyhow!("Failed to set Sora credentials in Redis: {}", e))?;
Ok(())
}
@@ -41,20 +41,17 @@ export class EngineApi extends ApiManager {
screenshot,
sceneMediaToken,
}: {
screenshot: File; // base64 encoded PNG
screenshot: string; // base64 encoded PNG
sceneMediaToken?: string;
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`;
const formData = new FormData();
formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec
if (sceneMediaToken) {
formData.append("scene_media_token", sceneMediaToken);
}
// for now ...
const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token
const response = await fetch(endpoint, {
method: "POST",
headers: {
@@ -96,10 +93,8 @@ export class EngineApi extends ApiManager {
additionalImages?: string[];
}): Promise<ApiResponse<string>> {
const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`;
// for now ...
const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID
const body = {
uuid_idempotency_token: uuidIdempotencyToken,
prompt,
snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token
additional_images: additionalImages,
@@ -107,7 +102,6 @@ export class EngineApi extends ApiManager {
const postResponse = await this.post<
{
uuid_idempotency_token: string;
prompt: string;
snapshot_media_token: string;
additional_images?: string[];
@@ -1,146 +0,0 @@
import { TransformControls } from "three/examples/jsm/Addons";
import * as THREE from "three";
export const FKHelperSphereName = "__FKHelperSphere__";
export class FKHelper {
private targetBoneSpheres: THREE.Object3D<THREE.Object3DEventMap>[];
private transformControls: TransformControls;
private raycaster: THREE.Raycaster;
private scene: THREE.Scene;
private skeletonHelper: THREE.SkeletonHelper | null = null;
private isControlInScene = false;
// Need the actual object to perform FK on
// This means the object must have bones
// Not all bones require FK right now
// Need a set of names to filter bones
// Use mixamo names
// Need a transform control (rotation mode) for the bones
// Need to disable the object transforms for bone transforms
// Need raycaster
constructor({
camera,
domElement,
scene,
onDragChange
}: {
camera: THREE.Camera;
domElement: HTMLElement;
scene: THREE.Scene;
onDragChange: (dragging: boolean) => void;
}) {
this.raycaster = new THREE.Raycaster();
this.scene = scene;
this.targetBoneSpheres = [];
this.transformControls = new TransformControls(camera, domElement);
this.transformControls.setSpace("local");
this.transformControls.setMode("rotate");
this.transformControls.setSize(0.5);
this.transformControls.addEventListener("dragging-changed", (event: any) => {
onDragChange(event.value);
});
}
// Set FK target
setTarget(target: THREE.Object3D) {
// TODO: Highlight all the necessary bones
this.clear();
target.traverse((child) => {
if (child.type !== "Bone") {
return;
}
// For each target bone, display a sphere to show the FK target and raycast intersect
const geometry = new THREE.SphereGeometry(1, 16, 16);
const material = new THREE.MeshBasicMaterial({ color: 0xff0000, depthTest: false, transparent: true });
const sphere = new THREE.Mesh(geometry, material);
sphere.name = FKHelperSphereName;
const uniformScale = 1.5;
sphere.scale.set(uniformScale, uniformScale, uniformScale);
child.add(sphere);
this.targetBoneSpheres.push(sphere);
});
this.skeletonHelper = new THREE.SkeletonHelper(target);
this.scene.add(this.skeletonHelper);
}
onMouseClick(mouse: THREE.Vector2) {
this.raycaster.setFromCamera(mouse, this.transformControls.camera);
const intersections = this.raycaster.intersectObjects(this.targetBoneSpheres, false);
if (intersections.length < 1) {
console.log("No intersections");
this.resetHighlight();
this.removeControls();
return;
}
// Attach the controls to the first bone
const firstBone = intersections[0];
this.highlightSphere(firstBone.object);
this.showControls(firstBone.object.parent!);
console.log("Showing bone", firstBone.object);
}
private showControls(object?: THREE.Object3D) {
if (object) {
this.transformControls.attach(object);
}
if (this.isControlInScene) {
return;
}
this.scene.add(this.transformControls);
this.isControlInScene = true;
}
private highlightSphere(sphere: THREE.Object3D) {
this.targetBoneSpheres.forEach((s) => {
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
s.material.opacity = 0.2
});
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
sphere.material.opacity = 1;
}
private resetHighlight() {
this.targetBoneSpheres.forEach((s) => {
// @ts-expect-error Material on object3d, but it's fine since this is a mesh
s.material.opacity = 1;
});
}
private removeControls() {
this.transformControls.detach();
if (this.isControlInScene) {
this.scene.remove(this.transformControls);
this.isControlInScene = false;
}
}
private removeSpheres() {
this.targetBoneSpheres.forEach((sphere) => {
this.scene.remove(sphere);
sphere.parent?.remove(sphere);
// @ts-expect-error Geometry on object3d, but it's fine since this is a mesh
sphere.geometry.dispose();
});
this.targetBoneSpheres = [];
}
clear() {
this.removeControls();
this.removeSpheres();
if (this.skeletonHelper) {
this.scene.remove(this.skeletonHelper!);
}
}
}
@@ -671,17 +671,7 @@ class Editor {
link.click();
}
const byteString = atob(base64Snapshot);
const mimeString = "image/png";
const ab = new ArrayBuffer(byteString.length);
const ia = new Uint8Array(ab);
for (let i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
const uuid = crypto.randomUUID(); // Generate a new UUID
const file = new File([ab], `${uuid}.png`, { type: mimeString });
return { base64Snapshot, file };
return base64Snapshot;
}
public async newScene(sceneTitleInput: string) {
@@ -960,8 +950,6 @@ class Editor {
}
deleteObject(uuid: string) {
this.mouse_controls?.clearFKVisuals();
this.mouse_controls?.removeTransformControls(true);
this.utils.deleteObject(uuid);
if (this.outliner_feature_flag) {
const result = this.sceneManager?.render_outliner(
@@ -993,26 +981,26 @@ class Editor {
this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
? 1024
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_9_16
CameraAspectRatio.VERTICAL_9_16
? 576
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
CameraAspectRatio.HORIZONTAL_3_2
? 900
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
CameraAspectRatio.VERTICAL_2_3
? 600
: 1000;
const height =
this.render_camera_aspect_ratio === CameraAspectRatio.HORIZONTAL_16_9
? 576
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_9_16
CameraAspectRatio.VERTICAL_9_16
? 1024
: this.render_camera_aspect_ratio ===
meraAspectRatio.HORIZONTAL_3_2
CameraAspectRatio.HORIZONTAL_3_2
? 600
: this.render_camera_aspect_ratio ===
meraAspectRatio.VERTICAL_2_3
CameraAspectRatio.VERTICAL_2_3
? 900
: 1000;
@@ -14,13 +14,6 @@ import { TransformControls } from "./TransformControls";
import { SceneManager, SceneObject } from "./scene_manager_api";
import { FreeCam } from "./free_cam";
import Stats from "three/examples/jsm/libs/stats.module.js";
import { FKHelper } from "./KinHelpers/FKHelper";
export enum KinMode {
FK,
IK,
NONE,
}
export class MouseControls {
camera: THREE.PerspectiveCamera | null;
@@ -54,12 +47,8 @@ export class MouseControls {
private isMovable: Function;
enable_stats: Function;
private kinMode: KinMode = KinMode.NONE;
private fkHelper: FKHelper;
private isBoneDragged: boolean = false;
constructor(
camera: THREE.PerspectiveCamera,
camera: THREE.PerspectiveCamera | null,
camera_person_mode: boolean,
cameraViewControls: FreeCam,
lockControls: PointerLockControls | undefined,
@@ -73,7 +62,7 @@ export class MouseControls {
mouse: THREE.Vector2 | undefined,
timeline_mouse: THREE.Vector2 | undefined,
raycaster: THREE.Raycaster | undefined,
control: TransformControls,
control: TransformControls | undefined,
outlinePass: OutlinePass | undefined,
scene: THREE.Scene,
publishSelect: Function,
@@ -112,20 +101,6 @@ export class MouseControls {
this.sceneManager = undefined;
this.isMovable = isMovable;
this.enable_stats = enable_stats;
this.fkHelper = new FKHelper({
camera: this.camera,
domElement: this.control.domElement,
scene: this.scene,
onDragChange: this.onFKControlsDragging.bind(this),
});
}
onFKControlsDragging(dragging: boolean) {
this.isBoneDragged = dragging;
}
clearFKVisuals() {
this.fkHelper.clear();
}
focus() {
@@ -150,26 +125,9 @@ export class MouseControls {
this.selected = [];
this.publishSelect();
}
this.hideTransformControls();
if (remove_outline) this.outlinePass.selectedObjects = [];
}
hideTransformControls() {
if (this.control == undefined) {
return;
}
this.control.detach();
this.scene.remove(this.control);
}
reattachTransformControls() {
if (this.control == undefined || this.selected == undefined) {
return;
}
this.control.attach(this.selected[0]);
this.scene.add(this.control);
if (remove_outline) this.outlinePass.selectedObjects = [];
}
selectObject(currentObject: THREE.Object3D) {
@@ -193,7 +151,7 @@ export class MouseControls {
}
onMouseDown(event: any) {
if ((event.button === 0 || event.button === 1) && this.isMovable() && !this.isBoneDragged) {
if ((event.button === 0 || event.button === 1) && this.isMovable()) {
this.isMouseClicked = true;
}
}
@@ -218,35 +176,6 @@ export class MouseControls {
return new Promise((resolve) => setTimeout(resolve, ms));
}
toggleFKMode() {
if (this.kinMode == KinMode.FK) {
this.fkHelper.clear();
this.kinMode = KinMode.NONE;
this.reattachTransformControls();
console.log("FK mode off");
return;
}
// Make sure we have an intersection
if (!(this.selected && this.selected.length > 0)) {
return;
}
// Make sure FK is supported only on character type objects
const firstSelection = this.selected[0];
if (!firstSelection.userData.isCharacter) {
return;
}
// FK is good to go
// Disable main transform controls
this.hideTransformControls();
this.kinMode = KinMode.FK;
this.fkHelper.setTarget(this.selected[0]);
console.log("FK mode on");
return;
}
async onkeydown(event: KeyboardEvent) {
if (hotkeysStatus.value.disabled) {
return;
@@ -269,7 +198,7 @@ export class MouseControls {
// transform
this.control?.setMode("translate");
return;
} else if (event.key === "r" && !event.ctrlKey) {
} else if (event.key === "r") {
// rotate
this.control?.setMode("rotate");
return;
@@ -277,8 +206,6 @@ export class MouseControls {
// scale
this.control?.setMode("scale");
return;
} else if (event.key === "k") {
this.toggleFKMode();
}
if ((event.ctrlKey || event.metaKey) && !this.isProcessing) {
@@ -371,11 +298,6 @@ export class MouseControls {
}
this.camera_last_pos.copy(camera_pos);
if (this.kinMode == KinMode.FK) {
this.fkHelper.onMouseClick(this.mouse);
return;
}
this.raycaster.setFromCamera(this.mouse, this.camera);
const interactable: any[] = [];
this.scene.children.forEach((child: THREE.Object3D) => {
@@ -287,6 +287,10 @@ export class TimeLine {
// testDeformBody(lastRig);
}
// Add a skeleton helper to the character
const skeletonHelper = new THREE.SkeletonHelper(obj);
this.scene.scene.add(skeletonHelper);
const object_uuid = obj.uuid;
this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.
@@ -25,7 +25,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
import { Button } from "~/components";
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
import { cameraAspectRatio, disableHotkeyInput, DomLevels, enableHotkeyInput } from "~/pages/PageEnigma/signals";
import { cameraAspectRatio } from "~/pages/PageEnigma/signals";
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
import Queue from "~/pages/PageEnigma/Queue/Queue";
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
@@ -191,7 +191,6 @@ export const PromptBox = () => {
},
});
};
reader.readAsDataURL(file);
});
}
@@ -243,18 +242,17 @@ export const PromptBox = () => {
const engineApi = new EngineApi();
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const snapshotResult = await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file,
sceneMediaToken: "",
});
await engineApi.enqueueImageGeneration({
prompt: prompt,
snapshotMediaToken: snapshotResult.data || "",
additionalImages: referenceImages.map((image) => image.mediaToken),
});
}
const snapshotResult = await engineApi.uploadSceneSnapshot({
screenshot: snapshot || "",
sceneMediaToken: "",
});
await engineApi.enqueueImageGeneration({
prompt: prompt,
snapshotMediaToken: snapshotResult.data || "",
additionalImages: referenceImages.map((image) => image.mediaToken),
});
setisEnqueueing(true);
try {
@@ -300,13 +298,12 @@ export const PromptBox = () => {
const handleSaveFrame = async () => {
if (editorEngine) {
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
if (snapshot) {
const engineApi = new EngineApi();
await engineApi.uploadSceneSnapshot({
screenshot: snapshot.file || "",
sceneMediaToken: "",
});
}
const engineApi = new EngineApi();
const result = await engineApi.uploadSceneSnapshot({
screenshot: snapshot || "",
sceneMediaToken: "",
});
console.log(result);
}
};
@@ -383,14 +380,6 @@ export const PromptBox = () => {
onChange={handleChange}
onPaste={handlePaste}
onKeyDown={handleKeyDown}
onFocus={() => {
console.log("disabling hotkeys")
disableHotkeyInput(DomLevels.INPUT);
}}
onBlur={() => {
console.log("enabling hotkeys")
enableHotkeyInput(DomLevels.INPUT);
}}
/>
</div>
<div className="mt-2 flex items-center justify-between gap-2">
@@ -51,7 +51,7 @@ export const Letterbox = ({
<div
id="letterbox"
className={twMerge(
"absolute left-0 top-0 flex h-full w-full justify-between pointer-events-none",
"absolute left-0 top-0 flex h-full w-full justify-between",
matteOri === MatteOrientation.TOP_BOTTOM ? "flex-col" : null,
)}
>