Compare commits

..
93 changed files with 1104 additions and 778 deletions
+2 -1
View File
@@ -75,4 +75,5 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt
# Development key. Permissible to check in.
NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL
STORYTELLER_ROOT="/home/tensor/code/storyteller/"
#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting.
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\n UPDATE model_weights\n SET\n title = ?,\n description_markdown = ?,\n maybe_thumbnail_token = ?,\n description_rendered_html = ?,\n creator_set_visibility = ?,\n version = version + 1\n WHERE token = ?\n LIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 6
},
"nullable": []
},
"hash": "4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\n UPDATE model_weights\n SET\n title = COALESCE(?, title),\n description_markdown = COALESCE(?, description_markdown),\n maybe_thumbnail_token = COALESCE(?, maybe_thumbnail_token),\n description_rendered_html = COALESCE(?, description_rendered_html),\n creator_set_visibility = COALESCE(?, creator_set_visibility),\n version = version + 1\n WHERE token = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 6
},
"nullable": []
},
"hash": "81df77a84fee48398e8a07e053b0393a0e341caf652c18c7160dd0a59247a19a"
}
Generated
+367 -331
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -17,6 +17,8 @@ path = "src/lib.rs"
# Internal
crockford = { path = "../../lib/crockford" }
errors = { path = "../../lib/errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
+6 -2
View File
@@ -13,15 +13,19 @@ name = "enums"
path = "src/lib.rs"
[dependencies]
# Internal
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
sqlx = { version = "0.7.2", features = [ "mysql", "sqlite", "runtime-tokio-native-tls", "chrono" ] }
sqlx = { version = "0.7.2", features = ["mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
sqlx-core = { version = "0.7.2" }
sqlx-mysql= { version = "0.7.2" }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
utoipa = { version = "4", features = ["chrono","actix_extras"] }
[dev-dependencies]
@@ -7,7 +7,7 @@ use strum::EnumIter;
use utoipa::ToSchema;
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize,ToSchema)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
pub enum WeightsType {
#[serde(rename = "hifigan_tt2")]
HifiganTacotron2,
@@ -106,7 +106,7 @@ mod tests {
#[test]
fn test_all_variants() {
let variants = WeightsType::all_variants();
assert_eq!(variants.len(), 7);
assert_eq!(variants.len(), 8);
assert!(variants.contains(&WeightsType::HifiganTacotron2));
assert!(variants.contains(&WeightsType::RvcV2));
assert!(variants.contains(&WeightsType::StableDiffusion15));
+3 -1
View File
@@ -17,6 +17,9 @@ path = "src/lib.rs"
# Internal
# (none)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
chrono = { version = "0.4.22", features = ["serde"] }
@@ -26,7 +29,6 @@ serde_derive = "1.0.125"
# MySql Database
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# Test serialization
+7 -1
View File
@@ -13,6 +13,13 @@ name = "tokens"
path = "src/lib.rs"
[dependencies]
# Internal
# (none)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
@@ -22,7 +29,6 @@ rand = "0.8.5"
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
sqlx-core = { version = "0.7.2" }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
utoipa = { version = "4", features = ["chrono","actix_extras"] }
+3 -1
View File
@@ -18,6 +18,9 @@ path = "src/main.rs"
container_common = { path = "../../lib/deprecated/container_common" }
easyenv = { path = "../../lib/easyenv" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
anyhow = "1.0.75"
# TODO/NB: Commenting out clap4 as it prevents windows builds
@@ -28,4 +31,3 @@ log = "0.4.14"
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+3 -1
View File
@@ -18,6 +18,9 @@ path = "src/main.rs"
container_common = { path = "../../lib/deprecated/container_common" }
easyenv = { path = "../../lib/easyenv" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
anyhow = "1.0.75"
# TODO/NB: Commenting out clap4 as it prevents windows builds
@@ -28,4 +31,3 @@ log = "0.4.14"
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+3 -1
View File
@@ -28,6 +28,9 @@ mysql_queries = { path = "../../lib/mysql_queries" }
storyteller_root = { path = "../../lib/storyteller_root" }
tokens = { path = "../../_types/tokens" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
log = "0.4.14"
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
@@ -39,7 +42,6 @@ tokio = { version = "1.6.0", features = ["macros"] }
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
elasticsearch = "8.5.0-alpha.1"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
@@ -1,22 +1,20 @@
use std::vec;
use buckets::public::media_files::original_file;
use log::info;
use sqlx::{MySql, Pool};
use tokens::tokens::users::UserToken;
use sqlx::{ MySql, Pool };
use enums::by_table::model_weights::{
weights_types::{WeightsType, self},
weights_category::{WeightsCategory, self},
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use errors::{ anyhow, AnyhowResult };
use enums::common::visibility::Visibility;
use errors::{anyhow, AnyhowResult};
use mysql_queries::queries::model_weights::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::seeding::users::HANASHI_USERNAME;
use mysql_queries::queries::model_weights::create_weight::{ create_weight, CreateModelWeightsArgs };
use tokens::tokens::model_weights::ModelWeightToken;
use enums::common::visibility::Visibility;
use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username;
pub async fn seed_weights_for_paging( mysql_pool: &Pool<MySql>,user_token: UserToken) -> AnyhowResult<()> {
let sd1_5_markdown_description = r#"
@@ -326,8 +324,8 @@ let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be7062
let args = CreateModelWeightsArgs {
token: &model_weight_token, // replace with actual ModelWeightToken
weights_type: weights_types, // replace with actual WeightsType
weights_category: weights_category, // replace with actual WeightsCategory
title: title,
weights_category, // replace with actual WeightsCategory
title,
maybe_thumbnail_token: Some(thumbnail_token.to_string()),
description_markdown: description,
description_rendered_html: description_rendered_html.to_string(),
@@ -342,11 +340,11 @@ let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be7062
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some(private_bucket_prefix),
maybe_private_bucket_extension: Some(private_bucket_extension),
cached_user_ratings_total_count: cached_user_ratings_total_count,
cached_user_ratings_positive_count: cached_user_ratings_positive_count,
cached_user_ratings_negative_count: cached_user_ratings_negative_count,
cached_user_ratings_total_count,
cached_user_ratings_positive_count,
cached_user_ratings_negative_count,
maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32),
version: version,
version,
mysql_pool: &mysql_pool, // replace with actual MySqlPool
};
+3 -1
View File
@@ -25,6 +25,9 @@ mimetypes = { path = "../../lib/files/mimetypes" }
mysql_queries = { path = "../../lib/mysql_queries" }
tokens = { path = "../../_types/tokens" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
log = "0.4.14"
@@ -34,7 +37,6 @@ log = "0.4.14"
tokio = { version = "1.6.0", features = ["macros"] }
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -24,6 +24,9 @@ mysql_queries = { path = "../../lib/mysql_queries" }
storyteller_root = { path = "../../lib/storyteller_root" }
tokens = { path = "../../_types/tokens" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
log = "0.4.14"
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
@@ -41,7 +44,6 @@ elasticsearch = "8.5.0-alpha.1"
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -18,12 +18,14 @@ config = { path = "../../lib/deprecated/config" }
easyenv = { path = "../../lib/easyenv" }
errors = { path = "../../lib/errors" }
# Workspace hack (faster compile yytimes)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
log = "0.4.20"
once_cell = "1.18.0"
rand = "0.8.5"
regex = "1.10.0"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Async Runtime
# NB: Not using Tokio here since everything is blocking so far.
+3 -1
View File
@@ -17,10 +17,12 @@ path = "src/main.rs"
errors = { path = "../../lib/errors" }
logging = { path = "../../lib/logging" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
clap = { version = "4.4.7", features = ["derive"] }
walkdir = "2.4.0"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -18,7 +18,9 @@ path = "src/main.rs"
errors = { path = "../../lib/errors" }
web_scrapers = { path = "../../lib/web_scrapers" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
tokio = { version = "1.6.0", features = ["macros"] }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+3 -1
View File
@@ -17,6 +17,9 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../lib/errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# Actix
actix-http = "3.4.0"
actix-rt = "2.9.0"
@@ -34,7 +37,6 @@ futures-util = "0.3.29"
# Serialization
serde = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
@@ -17,15 +17,16 @@ path = "src/lib.rs"
# Internal
buckets = { path = "../../../_types/buckets" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
log = "0.4.14"
# The "never-encode-slash" feature seems to have been removed:
# https://github.com/durch/rust-s3/pull/254
rust-s3 = { version = "0.33.0" }
rust-s3 = { version = "0.33.0" , default-features = false, features = ["tokio-rustls-tls"] }
# Runtime
# NB: We switched to Tokio from `async-std` for broader ecosystem support.
tokio = { version = "1.6.0", features = ["macros"] }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -18,9 +18,11 @@ path = "src/lib.rs"
container_common = { path = "../../deprecated/container_common" }
subprocess_common = { path = "../../subprocess_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
log = "0.4.14"
tempdir = "0.3.7"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -18,6 +18,9 @@ path = "src/lib.rs"
container_common = { path = "../../deprecated/container_common" }
easyenv = { path = "../../easyenv" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
once_cell = "1.18.0"
@@ -31,4 +34,3 @@ twitch_oauth2 = "0.6.0-rc.2"
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+3 -1
View File
@@ -19,11 +19,13 @@ easyenv = { path = "../easyenv" }
errors = { path = "../errors" }
server_environment = { path = "../server_environment" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
dotenv = "0.15.0"
hostname = "0.3.1"
log = "0.4.14"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -17,9 +17,11 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
# In-memory caching
lru_time_cache = "0.11.11"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+3 -1
View File
@@ -16,6 +16,9 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# Redis Database
r2d2_redis = "0.14.0"
@@ -23,7 +26,6 @@ r2d2_redis = "0.14.0"
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
# None yet
+2
View File
@@ -16,6 +16,8 @@ path = "src/lib.rs"
[dependencies]
# Internal
errors = { path = "../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
+3
View File
@@ -14,6 +14,9 @@ path = "src/lib.rs"
[dependencies]
# Internal
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
+3 -1
View File
@@ -17,11 +17,13 @@ path = "src/lib.rs"
# Internal
errors = { path = "../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
hmac = "0.10.0" # jwt signing
jwt = "0.13.0"
sha2 = "0.9.5" # hmac signing
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+7 -1
View File
@@ -15,8 +15,14 @@ name = "crockford"
path = "src/lib.rs"
[dependencies]
rand = "0.8.5"
# Internal
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
rand = "0.8.5"
[dev-dependencies]
# None yet
+7 -1
View File
@@ -13,9 +13,15 @@ name = "datetimes"
path = "src/lib.rs"
[dependencies]
# Internal
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
chrono = { version = "0.4.22", features = ["serde"] }
once_cell = "1.18.0"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# (none)
+3 -1
View File
@@ -18,7 +18,9 @@ path = "src/lib.rs"
# Internal
easyenv = { path = "../../../lib/easyenv" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
url = "2.4.1"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -21,10 +21,12 @@ rand = "0.8.5"
uuid = { version = "0.8.2", features = ["v4"] }
log = "0.4.14"
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# Serialization
serde = { version = "1.0.125", features = ["derive"] }
toml = "0.5.8"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
@@ -18,6 +18,9 @@ path = "src/lib.rs"
errors = { path = "../../errors" }
reusable_types = { path = "../../../_types/reusable_types" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
base64 = "0.21.0"
@@ -39,7 +42,6 @@ language-tags = "0.3.2" # Parsing single tags richly
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
actix-rt = "2.9.0"
+3 -1
View File
@@ -16,11 +16,13 @@ path = "src/lib.rs"
# Internal
storyteller_root = { path = "../storyteller_root" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
dotenv = "0.15.0"
env_logger = "0.10.1"
log = "0.4.14"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+6 -1
View File
@@ -10,6 +10,11 @@ path = "src/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
# Internal
errors = { path = "../errors" }
lettre = "0.11.2"
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
lettre = { version = "0.11.2" , default-features = false, features = ["tokio1-rustls-tls","smtp-transport", "pool", "hostname", "builder"] }
+2
View File
@@ -18,6 +18,8 @@ path = "src/lib.rs"
# Consider the dependencies here wisely.
# Ideally we have almost nothing as a dependency of this crate.
anyhow = "1.0.75"
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
+3 -1
View File
@@ -18,9 +18,11 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
zip = "0.6.6"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
# None yet
+4 -1
View File
@@ -16,11 +16,14 @@ path = "src/lib.rs"
[dependencies]
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
log = "0.4.20"
# External
# KEEP THIS TO A MINIMUM OR NIL.
log = "0.4.20"
[dev-dependencies]
# None yet
+3 -1
View File
@@ -17,12 +17,14 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
bcrypt = "0.8" # password hashes
data-encoding = "2.3.2"
md-5 = "0.9.1" # DO NOT USE FOR SERIOUS CRYPTO (we use this for gravatar)
ring = "0.17.0-alpha.10"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -19,9 +19,11 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
image = "0.24.1"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
testing = { path = "../../testing" }
+3 -2
View File
@@ -17,6 +17,9 @@ path = "src/lib.rs"
# Internal
errors = { path = "../../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
log = "0.4.14"
@@ -26,7 +29,5 @@ log = "0.4.14"
# Opus is the browser's default (only) recording codec: https://blog.addpipe.com/audio-constraints-getusermedia/
symphonia = { git = "https://github.com/pdeljanov/Symphonia.git", rev = "e1a70095c4409478c1ded499fb8dd4afa540fb90", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "opus", "vorbis", "wav"] }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
# None yet
+4 -1
View File
@@ -14,9 +14,12 @@ name = "mimetypes"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
infer = "0.11" # Mimetype detection
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -17,9 +17,11 @@ path = "src/lib.rs"
# errors = { path = "../../lib/errors" }
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
uuid = { version = "0.8.2", features = ["v4"] }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -22,9 +22,11 @@ container_common = { path = "../deprecated/container_common" }
errors = { path = "../errors" }
redis_common = { path = "../redis_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
log = "0.4.14"
r2d2_redis = "0.14.0"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+7 -1
View File
@@ -14,10 +14,16 @@ name = "logging"
path = "src/lib.rs"
[dependencies]
# Internal
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
dotenv = "0.15.0"
env_logger = "0.10.1"
log = "0.4.20"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+3 -1
View File
@@ -23,6 +23,9 @@ errors = { path = "../errors" }
reusable_types = { path = "../../_types/reusable_types" }
tokens = { path = "../../_types/tokens" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -44,5 +47,4 @@ tokio = { version = "1.6.0", features = ["full"] }
rand = "0.8"
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
@@ -1,38 +1,32 @@
#[cfg(test)]
mod tests {
use std::result;
use anyhow::Ok;
use chrono::offset;
use enums::by_table::model_weights::{ weights_types, weights_category };
use sqlx::{ MySqlPool, Acquire };
use rand::Rng;
use tokio;
use serial_test::serial;
use container_common::anyhow_result::AnyhowResult;
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use config::shared_constants::{ DEFAULT_MYSQL_CONNECTION_STRING };
use tokio;
use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
use container_common::anyhow_result::AnyhowResult;
// common tests
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken };
use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
use crate::queries::model_weights::create_weight::CreateModelWeightsArgs;
use crate::queries::model_weights::create_weight::create_weight;
use crate::queries::model_weights::get_weight::get_weight_by_token;
use crate::queries::model_weights::create_weight::CreateModelWeightsArgs;
use crate::queries::model_weights::delete_weights::{
delete_weights_as_user,
delete_weights_as_mod,
delete_weights_as_user,
undelete_weights_as_mod,
undelete_weights_as_user,
};
use crate::queries::model_weights::get_weight::get_weight_by_token;
use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username;
use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder;
use crate::queries::users::user::get_user_token_by_username::get_user_token_by_username;
+3 -1
View File
@@ -17,9 +17,11 @@ path = "src/lib.rs"
# Internal
errors = { path = "../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
bcrypt = "0.8" # password hashes
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
#hmac = "0.10.0" # jwt signing
#jwt = "0.13.0"
#sha2 = "0.9.5" # hmac signing
+3 -1
View File
@@ -17,6 +17,9 @@ path = "src/lib.rs"
# Internal
container_common = { path = "../deprecated/container_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
anyhow = "1.0.75"
@@ -27,4 +30,3 @@ serde_json = "1.0.108"
# TODO: Get rid of in favor of std::time
time = "0.3.30"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+7 -1
View File
@@ -14,8 +14,14 @@ name = "server_environment"
path = "src/lib.rs"
[dependencies]
# External
# Internal
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
# (None yet)
[dev-dependencies]
# None yet
+4 -1
View File
@@ -16,8 +16,11 @@ path = "src/lib.rs"
# Internal
errors = { path = "../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+7 -1
View File
@@ -13,9 +13,15 @@ name = "storyteller_root"
path = "src/lib.rs"
[dependencies]
# External
# Internal
# (None yet)
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
# (None yet)
[dev-dependencies]
serial_test = "2.0.0"
tempdir = "0.3.7"
+2
View File
@@ -18,6 +18,8 @@ path = "src/lib.rs"
# Internal
errors = { path = "../errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
+3 -1
View File
@@ -18,6 +18,9 @@ path = "src/lib.rs"
container_common = { path = "../deprecated/container_common" }
user_input_common = { path = "../user_input_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -29,4 +32,3 @@ unicode-segmentation = "1.10.1"
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+3 -1
View File
@@ -18,7 +18,9 @@ path = "src/lib.rs"
http_server_common = { path = "../deprecated/http_server_common" }
reusable_types = { path = "../../_types/reusable_types" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
actix-web = "4.4.0"
anyhow = "1.0.75"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+2 -1
View File
@@ -13,10 +13,11 @@ name = "user_input_common"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
once_cell = "1.18.0"
pulldown-cmark = "0.9.3" # Markdown
regex = "1.5.4"
unicode-segmentation = "1.10.1"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+3 -1
View File
@@ -17,6 +17,9 @@ path = "src/lib.rs"
enums = { path = "../../_types/enums" }
errors = { path = "../../lib/errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
# External
bytes = "1.3"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -35,7 +38,6 @@ reqwest = { version = "0.11", features = ["json"] } # HTTP Client
# Scraping
rss = "2.0.2"
scraper = "0.17.1"
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
# None yet
+45 -52
View File
@@ -16,160 +16,153 @@ publish = false
[dependencies]
actix-rt = { version = "2.9.0" }
ahash = { version = "0.8.6" }
base64 = { version = "0.21.5" }
base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] }
bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] }
block-buffer = { version = "0.9.0", default-features = false, features = ["block-padding"] }
bytemuck = { version = "1.14.0", default-features = false, features = ["extern_crate_alloc"] }
byteorder = { version = "1.5.0" }
chrono = { version = "0.4.22", features = ["alloc", "serde"] }
chrono = { version = "0.4.31", features = ["alloc", "serde"] }
combine = { version = "4.6.6", default-features = false, features = ["std", "tokio"] }
crossbeam-utils = { version = "0.8.16" }
deranged = { version = "0.3.9", default-features = false, features = ["powerfmt", "serde", "std"] }
deranged = { version = "0.3.10", default-features = false, features = ["powerfmt", "serde", "std"] }
digest = { version = "0.10.7", features = ["mac", "oid", "std"] }
either = { version = "1.9.0", features = ["serde"] }
flate2 = { version = "1.0.28" }
flume = { version = "0.11.0", default-features = false, features = ["async"] }
futures = { version = "0.3.29", features = ["thread-pool"] }
futures-channel = { version = "0.3.29", features = ["sink"] }
futures-core = { version = "0.3.29" }
futures-executor = { version = "0.3.29", features = ["thread-pool"] }
futures-executor = { version = "0.3.29", default-features = false, features = ["thread-pool"] }
futures-io = { version = "0.3.29" }
futures-sink = { version = "0.3.29" }
futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] }
generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] }
getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] }
getrandom-c65f7effa3be6d31 = { package = "getrandom", version = "0.1.16", default-features = false, features = ["std"] }
hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] }
hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] }
hashbrown = { version = "0.14.3", features = ["raw"] }
hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
hyper = { version = "0.14.27", features = ["client", "http1", "http2", "runtime", "server"] }
indexmap = { version = "2.1.0", features = ["serde"] }
lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] }
libc = { version = "0.2.150", features = ["align", "extra_traits"] }
libc = { version = "0.2.151", features = ["extra_traits"] }
log = { version = "0.4.20", default-features = false, features = ["std"] }
memchr = { version = "2.6.4", features = ["use_std"] }
miniz_oxide = { version = "0.7.1", features = ["simd"] }
mio = { version = "0.8.9", features = ["net", "os-ext"] }
mio = { version = "0.8.10", features = ["net", "os-ext"] }
native-tls = { version = "0.2.11", default-features = false, features = ["vendored"] }
num-integer = { version = "0.1.45", features = ["i128"] }
num-iter = { version = "0.1.43" }
num-traits = { version = "0.2.17", features = ["i128", "libm"] }
once_cell = { version = "1.18.0", features = ["unstable"] }
once_cell = { version = "1.19.0", features = ["unstable"] }
rand = { version = "0.8.5", features = ["small_rng"] }
rand_core = { version = "0.6.4", default-features = false, features = ["std"] }
regex = { version = "1.10.2" }
regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
regex-syntax = { version = "0.8.2" }
reqwest = { version = "0.11.22", features = ["gzip", "json", "native-tls", "stream"] }
reqwest = { version = "0.11.22", features = ["blocking", "gzip", "json", "native-tls", "rustls-tls", "stream"] }
rustls = { version = "0.21.10", features = ["dangerous_configuration"] }
serde = { version = "1.0.193", features = ["alloc", "derive", "rc"] }
serde_json = { version = "1.0.108", features = ["raw_value"] }
sha1 = { version = "0.10.6" }
sha2 = { version = "0.10.8" }
simd-adler32 = { version = "0.3.7" }
socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
sqlx = { version = "0.7.2", features = ["chrono", "mysql", "runtime-tokio-native-tls", "sqlite", "tls-rustls"] }
sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] }
sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] }
sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] }
time = { version = "0.3.30", features = ["macros", "serde-well-known"] }
tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
tokio = { version = "1.35.0", features = ["full"] }
tokio-rustls = { version = "0.24.1" }
tokio-stream = { version = "0.1.14", features = ["fs"] }
tracing = { version = "0.1.40", features = ["log"] }
twitch_oauth2 = { version = "0.6.1", features = ["reqwest_client"] }
unicode-bidi = { version = "0.3.13" }
unicode-bidi = { version = "0.3.14" }
unicode-normalization = { version = "0.1.22" }
url = { version = "2.4.1", features = ["serde"] }
url = { version = "2.5.0", features = ["serde"] }
void = { version = "1.0.2" }
zip = { version = "0.6.6" }
[build-dependencies]
ahash = { version = "0.8.6" }
base64 = { version = "0.21.5" }
base64ct = { version = "1.6.0", default-features = false, features = ["alloc"] }
bitflags = { version = "2.4.1", default-features = false, features = ["serde", "std"] }
byteorder = { version = "1.5.0" }
cc = { version = "1.0.83", default-features = false, features = ["parallel"] }
chrono = { version = "0.4.22", features = ["alloc", "serde"] }
chrono = { version = "0.4.31", features = ["alloc", "serde"] }
crossbeam-utils = { version = "0.8.16" }
deranged = { version = "0.3.10", default-features = false, features = ["powerfmt", "serde", "std"] }
derive_more = { version = "0.99.17" }
digest = { version = "0.10.7", features = ["mac", "oid", "std"] }
either = { version = "1.9.0", features = ["serde"] }
flume = { version = "0.11.0", default-features = false, features = ["async"] }
flate2 = { version = "1.0.28" }
futures-channel = { version = "0.3.29", features = ["sink"] }
futures-core = { version = "0.3.29" }
futures-executor = { version = "0.3.29", features = ["thread-pool"] }
futures-io = { version = "0.3.29" }
futures-sink = { version = "0.3.29" }
futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] }
generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths"] }
getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.11", default-features = false, features = ["std"] }
hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] }
hashbrown = { version = "0.14.3", features = ["raw"] }
heck = { version = "0.4.1", features = ["unicode"] }
hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
indexmap = { version = "2.1.0", features = ["serde"] }
lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] }
libc = { version = "0.2.150", features = ["align", "extra_traits"] }
libc = { version = "0.2.151", features = ["extra_traits"] }
log = { version = "0.4.20", default-features = false, features = ["std"] }
memchr = { version = "2.6.4", features = ["use_std"] }
mio = { version = "0.8.9", features = ["net", "os-ext"] }
native-tls = { version = "0.2.11", default-features = false, features = ["vendored"] }
miniz_oxide = { version = "0.7.1", features = ["simd"] }
mio = { version = "0.8.10", features = ["net", "os-ext"] }
num-integer = { version = "0.1.45", features = ["i128"] }
num-iter = { version = "0.1.43" }
num-traits = { version = "0.2.17", features = ["i128", "libm"] }
once_cell = { version = "1.18.0", features = ["unstable"] }
once_cell = { version = "1.19.0", features = ["unstable"] }
rand = { version = "0.8.5", features = ["small_rng"] }
rand_core = { version = "0.6.4", default-features = false, features = ["std"] }
regex = { version = "1.10.2" }
regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
regex-syntax = { version = "0.8.2" }
rustls = { version = "0.21.10", features = ["dangerous_configuration"] }
serde = { version = "1.0.193", features = ["alloc", "derive", "rc"] }
serde_json = { version = "1.0.108", features = ["raw_value"] }
sha1 = { version = "0.10.6" }
sha2 = { version = "0.10.8" }
sqlx-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "any", "chrono", "json", "migrate", "offline"] }
sqlx-macros = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] }
sqlx-macros-core = { version = "0.7.2", features = ["_rt-tokio", "_tls-native-tls", "_tls-rustls", "chrono", "json", "migrate", "mysql", "sqlite"] }
sqlx-mysql = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] }
sqlx-sqlite = { version = "0.7.2", default-features = false, features = ["any", "chrono", "json", "migrate", "offline"] }
simd-adler32 = { version = "0.3.7" }
syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit-mut"] }
syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.39", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.40", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
time = { version = "0.3.30", features = ["macros", "serde-well-known"] }
time-macros = { version = "0.2.15", default-features = false, features = ["formatting", "parsing", "serde"] }
tokio = { version = "1.34.0", features = ["fs", "io-util", "macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
tokio = { version = "1.35.0", features = ["full"] }
tokio-stream = { version = "0.1.14", features = ["fs"] }
tracing = { version = "0.1.40", features = ["log"] }
unicode-bidi = { version = "0.3.13" }
unicode-bidi = { version = "0.3.14" }
unicode-normalization = { version = "0.1.22" }
url = { version = "2.4.1", features = ["serde"] }
url = { version = "2.5.0", features = ["serde"] }
zip = { version = "0.6.6" }
[target.x86_64-unknown-linux-gnu.dependencies]
openssl = { version = "0.10.59", features = ["vendored"] }
openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] }
rustix = { version = "0.38.26", features = ["fs", "termios"] }
hyper-rustls = { version = "0.24.2", default-features = false, features = ["http1", "http2", "logging", "tls12", "webpki-tokio"] }
openssl = { version = "0.10.61", features = ["vendored"] }
openssl-sys = { version = "0.9.97", default-features = false, features = ["vendored"] }
rustix = { version = "0.38.28", features = ["fs", "termios"] }
spin = { version = "0.9.8" }
[target.x86_64-unknown-linux-gnu.build-dependencies]
openssl = { version = "0.10.59", features = ["vendored"] }
openssl-sys = { version = "0.9.95", default-features = false, features = ["vendored"] }
rustix = { version = "0.38.26", features = ["fs", "termios"] }
socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
rustix = { version = "0.38.28", features = ["fs", "termios"] }
spin = { version = "0.9.8" }
[target.x86_64-apple-darwin.dependencies]
rustix = { version = "0.38.26", features = ["fs", "termios"] }
hyper-rustls = { version = "0.24.2", default-features = false, features = ["http1", "http2", "logging", "tls12", "webpki-tokio"] }
rustix = { version = "0.38.28", features = ["fs", "termios"] }
spin = { version = "0.9.8" }
[target.x86_64-apple-darwin.build-dependencies]
rustix = { version = "0.38.26", features = ["fs", "termios"] }
socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
rustix = { version = "0.38.28", features = ["fs", "termios"] }
spin = { version = "0.9.8" }
[target.x86_64-pc-windows-msvc.dependencies]
hyper-rustls = { version = "0.24.2", default-features = false, features = ["http1", "http2", "logging", "tls12", "webpki-tokio"] }
spin = { version = "0.9.8" }
winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] }
winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Com", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming", "Win32_UI_Shell"] }
[target.x86_64-pc-windows-msvc.build-dependencies]
socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
spin = { version = "0.9.8" }
winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winreg", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming"] }
winapi = { version = "0.3.9", default-features = false, features = ["consoleapi", "errhandlingapi", "fibersapi", "fileapi", "handleapi", "knownfolders", "memoryapi", "minwinbase", "minwindef", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "winerror", "winnt", "winsock2", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
windows-sys = { version = "0.48.0", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Networking_WinSock", "Win32_Security_Authentication_Identity", "Win32_Security_Credentials", "Win32_Security_Cryptography", "Win32_Storage_FileSystem", "Win32_System_Com", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Registry", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_System_Time", "Win32_System_WindowsProgramming", "Win32_UI_Shell"] }
### END HAKARI SECTION
+3 -1
View File
@@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
reusable_types = { path = "../../../_types/reusable_types" }
tokens = { path = "../../../_types/tokens" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -42,4 +45,3 @@ tokio = { version = "1.6.0", features = ["macros"] }
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+3 -1
View File
@@ -33,6 +33,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
reusable_types = { path = "../../../_types/reusable_types" }
subprocess_common = { path = "../../../lib/subprocess_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -60,4 +63,3 @@ hyper = { version = "0.14.24" }
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -41,6 +41,9 @@ testing = { path = "../../../lib/testing" }
tokens = { path = "../../../_types/tokens" }
tts_common = { path = "../../../lib/tts_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# Vendored code
# newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" }
@@ -82,4 +85,3 @@ serde_json = "1.0.108"
# The initial round of stabilizations for the async/await language feature in Rust 1.39 did not
# include support for async fn in traits.
async-trait = "0.1.58"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+3 -1
View File
@@ -38,6 +38,9 @@ subprocess_common = { path = "../../../lib/subprocess_common" }
tokens = { path = "../../../_types/tokens" }
tts_common = { path = "../../../lib/tts_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# Vendored code
newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" }
@@ -78,4 +81,3 @@ serde_json = "1.0.108"
# The initial round of stabilizations for the async/await language feature in Rust 1.39 did not
# include support for async fn in traits.
async-trait = "0.1.58"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -82,6 +82,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit
TT2_LEGACY_ROOT_DIRECTORY="/models/tts"
TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6"
TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df"
TT2_AS_MEDIA_FILES="true"
# ================================= #
# #
@@ -5,14 +5,17 @@ use log::{error, warn};
use sqlx::MySqlPool;
use tempdir::TempDir;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::get_media_file::get_media_file;
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
use mysql_queries::queries::voice_conversion::results::get_voice_conversion_result_for_inference::get_voice_conversion_result_for_inference;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::media_uploads::MediaUploadToken;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -23,86 +26,47 @@ pub struct AudioFile {
pub filesystem_path: PathBuf,
}
pub async fn download_audio_file(
audio_source: &LipsyncAnimationAudioSource,
public_bucket_client: &BucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
temp_dir_creator: &ScopedTempDirCreator,
work_temp_dir: &TempDir,
mysql_pool: &MySqlPool,
) -> Result<AudioFile, ProcessSingleJobError> {
pub struct DownloadAudioFileArgs<'a> {
pub audio_source: &'a LipsyncAnimationAudioSource,
pub public_bucket_client: &'a BucketClient,
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
pub job: &'a AvailableInferenceJob,
pub temp_dir_creator: &'a ScopedTempDirCreator,
pub work_temp_dir: &'a TempDir,
pub mysql_pool: &'a MySqlPool,
}
pub async fn download_audio_file(args: DownloadAudioFileArgs<'_>) -> Result<AudioFile, ProcessSingleJobError> {
let bucket_object_path;
match audio_source {
match args.audio_source {
LipsyncAnimationAudioSource::F(media_file_token) => {
// TODO(bt, 2023-09-08): Implement
return Err(ProcessSingleJobError::NotYetImplemented)
bucket_object_path = from_media_file(media_file_token, &args).await?;
}
LipsyncAnimationAudioSource::U(media_upload_token) => {
let media_upload_token = MediaUploadToken::new_from_str(media_upload_token);
let media_upload_result = get_media_upload_for_inference(
&media_upload_token,
mysql_pool
).await;
let media_upload_result = match media_upload_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media upload: {:?}", media_upload_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find media upload: {:?}", media_upload_token)))
}
Err(e) => {
error!("could not query media upload: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash);
bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf();
bucket_object_path = from_media_upload(media_upload_token, &args).await?;
}
LipsyncAnimationAudioSource::T(tts_result_token) => {
// TODO(bt, 2023-09-08): Implement
LipsyncAnimationAudioSource::T(_tts_result_token) => {
// NB(bt,2023-12-08): This will likely never be implemented now that TTS is on media files.
return Err(ProcessSingleJobError::NotYetImplemented)
}
LipsyncAnimationAudioSource::V(voice_conversion_result_token) => {
let voice_conversion_result = get_voice_conversion_result_for_inference(
voice_conversion_result_token,
false,
&mysql_pool,
).await;
let voice_conversion_result = match voice_conversion_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find voice conversion result: {:?}", voice_conversion_result_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token)))
}
Err(e) => {
error!("could not query voice conversion result: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash);
bucket_object_path = result_bucket_path.to_full_object_pathbuf();
bucket_object_path = from_voice_conversion_result(voice_conversion_result_token, &args).await?;
}
}
let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin");
let downloaded_filesystem_path = args.work_temp_dir.path().join("audio.bin");
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "audio file",
final_filesystem_file_path: &downloaded_filesystem_path,
bucket_object_path: &bucket_object_path,
bucket_client: public_bucket_client,
job_progress_reporter,
bucket_client: args.public_bucket_client,
job_progress_reporter: args.job_progress_reporter,
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &temp_dir_creator,
job_id: args.job.id.0,
scoped_tempdir_creator: &args.temp_dir_creator,
maybe_existing_file_minimum_size_required: None,
}).await?;
@@ -110,3 +74,82 @@ pub async fn download_audio_file(
filesystem_path: downloaded_filesystem_path,
})
}
async fn from_media_file(media_file_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result<PathBuf, ProcessSingleJobError> {
let media_file_token = MediaFileToken::new_from_str(media_file_token);
let media_file_result = get_media_file(
&media_file_token,
true,
args.mysql_pool
).await;
let media_file_result = match media_file_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media file: {:?}", media_file_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find media file : {:?}", media_file_token)))
}
Err(e) => {
error!("could not query media file: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
&media_file_result.public_bucket_directory_hash,
media_file_result.maybe_public_bucket_prefix.as_deref(),
media_file_result.maybe_public_bucket_extension.as_deref());
Ok(media_file_bucket_path.to_full_object_pathbuf())
}
async fn from_media_upload(media_upload_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result<PathBuf, ProcessSingleJobError> {
let media_upload_token = MediaUploadToken::new_from_str(media_upload_token);
let media_upload_result = get_media_upload_for_inference(
&media_upload_token,
args.mysql_pool
).await;
let media_upload_result = match media_upload_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media upload: {:?}", media_upload_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find media upload: {:?}", media_upload_token)))
}
Err(e) => {
error!("could not query media upload: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash);
Ok(media_upload_bucket_path.to_full_object_pathbuf())
}
async fn from_voice_conversion_result(voice_conversion_result_token: &str, args: &DownloadAudioFileArgs<'_>) -> Result<PathBuf, ProcessSingleJobError> {
let voice_conversion_result = get_voice_conversion_result_for_inference(
voice_conversion_result_token,
false,
&args.mysql_pool,
).await;
let voice_conversion_result = match voice_conversion_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find voice conversion result: {:?}", voice_conversion_result_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find voice conversion result: {:?}", voice_conversion_result_token)))
}
Err(e) => {
error!("could not query voice conversion result: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let result_bucket_path = VoiceConversionResultOriginalFilePath::from_object_hash(&voice_conversion_result.public_bucket_hash);
Ok(result_bucket_path.to_full_object_pathbuf())
}
@@ -5,12 +5,15 @@ use log::{error, warn};
use sqlx::MySqlPool;
use tempdir::TempDir;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::get_media_file::get_media_file;
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::media_uploads::MediaUploadToken;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -21,59 +24,40 @@ pub struct ImageFile {
pub filesystem_path: PathBuf,
}
pub async fn download_image_file(
image_source: &LipsyncAnimationImageSource,
public_bucket_client: &BucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
temp_dir_creator: &ScopedTempDirCreator,
work_temp_dir: &TempDir,
mysql_pool: &MySqlPool,
) -> Result<ImageFile, ProcessSingleJobError> {
pub struct DownloadImageFileArgs<'a> {
pub image_source: &'a LipsyncAnimationImageSource,
pub public_bucket_client: &'a BucketClient,
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
pub job: &'a AvailableInferenceJob,
pub temp_dir_creator: &'a ScopedTempDirCreator,
pub work_temp_dir: &'a TempDir,
pub mysql_pool: &'a MySqlPool,
}
pub async fn download_image_file(args: DownloadImageFileArgs<'_>) -> Result<ImageFile, ProcessSingleJobError> {
let bucket_object_path;
match image_source {
match args.image_source {
LipsyncAnimationImageSource::F(media_file_token) => {
// TODO(bt, 2023-09-08): Implement
return Err(ProcessSingleJobError::NotYetImplemented)
bucket_object_path = from_media_file(media_file_token, &args).await?;
}
LipsyncAnimationImageSource::U(media_upload_token) => {
let media_upload_token = MediaUploadToken::new_from_str(media_upload_token);
let media_upload_result = get_media_upload_for_inference(
&media_upload_token,
mysql_pool
).await;
let media_upload_result = match media_upload_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media upload: {:?}", media_upload_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find media upload: {:?}", media_upload_token)))
}
Err(e) => {
error!("could not query media upload: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash);
bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf();
bucket_object_path = from_media_upload(media_upload_token, &args).await?;
}
}
let downloaded_filesystem_path = work_temp_dir.path().join("image.bin");
let downloaded_filesystem_path = args.work_temp_dir.path().join("image.bin");
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "image file",
final_filesystem_file_path: &downloaded_filesystem_path,
bucket_object_path: &bucket_object_path,
bucket_client: public_bucket_client,
job_progress_reporter,
bucket_client: args.public_bucket_client,
job_progress_reporter: args.job_progress_reporter,
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &temp_dir_creator,
job_id: args.job.id.0,
scoped_tempdir_creator: &args.temp_dir_creator,
maybe_existing_file_minimum_size_required: None,
}).await?;
@@ -81,3 +65,56 @@ pub async fn download_image_file(
filesystem_path: downloaded_filesystem_path,
})
}
async fn from_media_file(media_file_token: &str, args: &DownloadImageFileArgs<'_>) -> Result<PathBuf, ProcessSingleJobError> {
let media_file_token = MediaFileToken::new_from_str(media_file_token);
let media_file_result = get_media_file(
&media_file_token,
true,
args.mysql_pool
).await;
let media_file_result = match media_file_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media file : {:?}", media_file_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find media file : {:?}", media_file_token)))
}
Err(e) => {
error!("could not query media file : {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
&media_file_result.public_bucket_directory_hash,
media_file_result.maybe_public_bucket_prefix.as_deref(),
media_file_result.maybe_public_bucket_extension.as_deref());
Ok(media_file_bucket_path.to_full_object_pathbuf())
}
async fn from_media_upload(media_upload_token: &str, args: &DownloadImageFileArgs<'_>) -> Result<PathBuf, ProcessSingleJobError> {
let media_upload_token = MediaUploadToken::new_from_str(media_upload_token);
let media_upload_result = get_media_upload_for_inference(
&media_upload_token,
args.mysql_pool
).await;
let media_upload_result = match media_upload_result {
Ok(Some(result)) => result,
Ok(None) => {
warn!("could not find media upload: {:?}", media_upload_token);
return Err(ProcessSingleJobError::from_anyhow_error(
anyhow!("could not find media upload: {:?}", media_upload_token)))
}
Err(e) => {
error!("could not query media upload: {:?}", e);
return Err(ProcessSingleJobError::from_anyhow_error(e))
}
};
let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash);
Ok(media_upload_bucket_path.to_full_object_pathbuf())
}
@@ -19,8 +19,8 @@ use tokens::tokens::users::UserToken;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error;
use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file;
use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file;
use crate::job::job_types::lipsync::sad_talker::download_audio_file::{download_audio_file, DownloadAudioFileArgs};
use crate::job::job_types::lipsync::sad_talker::download_image_file::{download_image_file, DownloadImageFileArgs};
use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image;
use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs;
use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job;
@@ -97,27 +97,27 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess
// ==================== QUERY AND DOWNLOAD FILES ==================== //
let audio_path = download_audio_file(
&job_args.audio_source,
&args.job_dependencies.buckets.public_bucket_client,
&mut job_progress_reporter,
let audio_path = download_audio_file(DownloadAudioFileArgs {
audio_source: &job_args.audio_source,
public_bucket_client: &args.job_dependencies.buckets.public_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
&work_temp_dir,
&deps.db.mysql_pool
).await?;
temp_dir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
work_temp_dir: &work_temp_dir,
mysql_pool: &deps.db.mysql_pool
}).await?;
info!("Audio file: {:?}", audio_path.filesystem_path);
info!("Downloaded audio file: {:?}", audio_path.filesystem_path);
let image_path = download_image_file(
&job_args.image_source,
&args.job_dependencies.buckets.public_bucket_client,
&mut job_progress_reporter,
let image_path = download_image_file(DownloadImageFileArgs {
image_source: &job_args.image_source,
public_bucket_client: &args.job_dependencies.buckets.public_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
&work_temp_dir,
&deps.db.mysql_pool
).await?;
temp_dir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
work_temp_dir: &work_temp_dir,
mysql_pool: &deps.db.mysql_pool
}).await?;
info!("Downloaded image file: {:?}", image_path.filesystem_path);
@@ -7,16 +7,13 @@ use anyhow::anyhow;
use log::{error, info};
use tempdir::TempDir;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use errors::AnyhowResult;
use filesys::check_file_exists::check_file_exists;
use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
use filesys::safe_delete_temp_file::safe_delete_temp_file;
use hashing::sha256::sha256_hash_string::sha256_hash_string;
use mysql_queries::column_types::vocoder_type::VocoderType;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord;
use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType};
use tts_common::clean_symbols::clean_symbols;
use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic;
use tts_common::text_pipelines::text_pipeline_type::TextPipelineType;
@@ -1,11 +1,10 @@
use std::path::Path;
use anyhow::anyhow;
use anyhow::anyhow;
use log::info;
use tempdir::TempDir;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
use filesys::safe_delete_temp_file::safe_delete_temp_file;
@@ -15,8 +14,8 @@ use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReport
use mysql_queries::column_types::vocoder_type::VocoderType;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs};
use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType;
use mysql_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result;
use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::process_job::FileMetadata;
@@ -61,16 +61,14 @@ pub async fn process_create_voice(
let creator_ip_address = &job.creator_ip_address;
let creator_user_token: UserToken;
match &job.maybe_creator_user_token {
let creator_user_token = match &job.maybe_creator_user_token {
Some(token) => {
creator_user_token = UserToken::new_from_str(token);
UserToken::new_from_str(token)
},
None => {
return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
}
}
};
// STEP 1. SETUP A TEMP DIRECTORY
let work_temp_dir = format!("/tmp/temp_zeroshot_create_voice_{}", job.id.0);
@@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" }
jobs_common = { path = "../../../lib/jobs_common" }
mysql_queries = { path = "../../../lib/mysql_queries" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -52,4 +55,3 @@ hyper = { version = "0.14.24" }
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -27,6 +27,9 @@ memory_caching = { path = "../../../lib/caching/memory_caching" }
mysql_queries = { path = "../../../lib/mysql_queries" }
tts_common = { path = "../../../lib/tts_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# Vendored code
newrelic-telemetry = { path = "../../../vendor/newrelic-telemetry-sdk-rust" }
@@ -58,4 +61,3 @@ hyper = { version = "0.14.24" }
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -26,6 +26,9 @@ hashing = { path = "../../../lib/files/hashing" }
jobs_common = { path = "../../../lib/jobs_common" }
mysql_queries = { path = "../../../lib/mysql_queries" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -52,4 +55,3 @@ hyper = { version = "0.14.24" }
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -24,6 +24,9 @@ google_drive_common = { path = "../../../lib/api_clients/google_drive_common" }
jobs_common = { path = "../../../lib/jobs_common" }
mysql_queries = { path = "../../../lib/mysql_queries" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
chrono = { version = "0.4.22", features = ["serde"] }
@@ -50,4 +53,3 @@ hyper = { version = "0.14.24" }
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -25,6 +25,9 @@ tokens = { path = "../../../_types/tokens" }
tts_common = { path = "../../../lib/tts_common" }
twitch_common = { path = "../../../lib/api_clients/twitch_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
base64 = "0.21.0"
@@ -81,7 +84,6 @@ toml = "0.5.8"
# Text processing
pulldown-cmark = "0.9.3"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[build-dependencies]
# NB: Only using diesel for migrations support
+4 -2
View File
@@ -21,6 +21,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
reusable_types = { path = "../../../_types/reusable_types" }
url_config = { path = "../../../lib/url_config" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
base64 = "0.21.0"
@@ -42,7 +45,7 @@ serde_json = "1.0.108"
toml = "0.5.8"
# Money
async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] }
async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper-rustls-webpki", "connect", "billing", "checkout", "webhook-events"] }
# Async traits
# NB: note: `async` trait functions are not currently supported
@@ -51,7 +54,6 @@ async-stripe = { version = "0.25.2", default-features = false, features = ["runt
async-trait = "0.1.58"
sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[dev-dependencies]
mockall = "0.11.3"
@@ -1,60 +1,49 @@
//! These routes are recommended, but do not have to be used by consumers of the billing system.
use actix_web::{App, HttpResponse, web};
use actix_web::body::MessageBody;
use actix_web::dev::{ServiceFactory, ServiceRequest, ServiceResponse};
use actix_web::error::Error;
use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_json_handler::stripe_create_checkout_session_json_handler;
use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_redirect_handler::stripe_create_checkout_session_redirect_handler;
use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_json_handler::stripe_create_customer_portal_session_json_handler;
use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_redirect_handler::stripe_create_customer_portal_session_redirect_handler;
use crate::stripe::http_endpoints::webhook::stripe_webhook_handler::stripe_webhook_handler;
use actix_web::body::MessageBody;
use actix_web::dev::{ServiceFactory, ServiceRequest, ServiceResponse};
use actix_web::error::Error;
use log::warn;
use crate::users::http_endpoints::list_active_user_subscriptions_handler::list_active_user_subscriptions_handler;
pub fn add_suggested_stripe_billing_routes<T, B> (app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<B>,
Error = Error,
InitError = (),
>,
pub fn add_suggested_stripe_billing_routes<T, B>(app: App<T>) -> App<T>
where
B: MessageBody,
T: ServiceFactory<ServiceRequest, Config = (), Response = ServiceResponse<B>, Error = Error, InitError = ()>,
{
app.service(web::scope("/v1")
.service(web::scope("/billing")
.service(web::resource("/active_subscriptions")
.route(web::get().to(list_active_user_subscriptions_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::scope("/stripe")
.service(web::resource("/webhook")
.route(web::post().to(stripe_webhook_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::scope("/checkout")
.service(web::resource("/create_redirect")
.route(web::post().to(stripe_create_checkout_session_json_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/redirect")
.route(web::get().to(stripe_create_checkout_session_redirect_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::scope("/portal")
.service(web::resource("/create_redirect")
.route(web::post().to(stripe_create_customer_portal_session_json_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/redirect")
.route(web::get().to(stripe_create_customer_portal_session_redirect_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
)
warn!("Adding suggested Stripe billing routes to app.");
app.service(
web::scope("/v1")
.service(web::scope("/billing")
.service(web::resource("/active_subscriptions")
.route(web::get().to(list_active_user_subscriptions_handler))
.route(web::head().to(|| HttpResponse::Ok()))))
.service(web::scope("/stripe")
.service(web::resource("/webhook")
.route(web::post().to(stripe_webhook_handler))
.route(web::head().to(|| HttpResponse::Ok())))
.service(web::scope("/checkout")
.service(web::resource("/create_redirect")
.route(web::post().to(stripe_create_checkout_session_json_handler))
.route(web::head().to(|| HttpResponse::Ok())))
.service(web::resource("/redirect")
.route(web::get().to(stripe_create_checkout_session_redirect_handler))
.route(web::head().to(|| HttpResponse::Ok()))))
.service(web::scope("/portal")
.service(web::resource("/create_redirect")
.route(web::post().to(stripe_create_customer_portal_session_json_handler))
.route(web::head().to(|| HttpResponse::Ok())))
.service(web::resource("/redirect")
.route(web::get().to(stripe_create_customer_portal_session_redirect_handler))
.route(web::head().to(|| HttpResponse::Ok()))))),
)
}
@@ -14,5 +14,5 @@ path = "src/lib.rs"
[dependencies]
# External
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+3 -1
View File
@@ -31,6 +31,9 @@ server_environment = { path = "../../../lib/server_environment" }
tokens = { path = "../../../_types/tokens" }
user_input_common = { path = "../../../lib/user_input_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
base64 = "0.21.0"
@@ -65,4 +68,3 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls",
r2d2_redis = "0.14.0"
byteorder = "1.5.0"
strum_macros = "0.25.3"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+3 -1
View File
@@ -20,6 +20,9 @@ easyenv = { path = "../../../lib/easyenv" }
errors = { path = "../../../lib/errors" }
reusable_types = { path = "../../../_types/reusable_types" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
chrono = { version = "0.4.22", features = ["serde"] }
dotenv = "0.15.0"
@@ -45,4 +48,3 @@ actix-web = "4.4.0"
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.108"
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+10 -5
View File
@@ -51,7 +51,7 @@ url_config = { path = "../../../lib/url_config" }
user_input_common = { path = "../../../lib/user_input_common" }
# Internal Components
billing_component = { path = "../../plugins/billing" }
billing_component = { path = "../../plugins/billing", optional = true }
users_component = { path = "../../plugins/users" }
# Workspace hack (faster compile times)
@@ -90,7 +90,7 @@ language-tags = "0.3.2" # Parsing single tags richly
# HTTP
# TODO/NB: This is just for Twitch, but we might need it later.
reqwest = { version = "0.11.4" }
reqwest = { version = "0.11.4", default-features = false, features = ["json", "blocking", "rustls-tls"] }
native-tls = { version = "0.2.8", features = ["vendored"] }
# Runtime
@@ -123,7 +123,7 @@ infer = "0.5.0"
# The "never-encode-slash" feature seems to have been removed:
# https://github.com/durch/rust-s3/pull/254
rust-s3 = { version = "0.33.0" }
rust-s3 = { version = "0.33.0" , default-features = false, features = ["tokio-rustls-tls"] }
actix-cors = "0.6.4"
actix-http = "3.4.0"
@@ -139,7 +139,7 @@ jwt = "0.13.0"
hyper = { version = "0.14.24" }
# Money
async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] }
#async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper-rustls-webpki", "connect", "billing", "checkout", "webhook-events"] }
# Crypto
bcrypt = "0.8" # password hashes
@@ -163,7 +163,6 @@ pulldown-cmark = "0.9.3"
# include support for async fn in traits.
async-trait = "0.1.58"
my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" }
utoipa = { version = "4", features = ["chrono","actix_extras"] }
utoipa-swagger-ui = { version = "5", features = ["actix-web"] }
@@ -173,3 +172,9 @@ diesel = { version = "2.1.3", features = ["mysql"] }
[dev-dependencies]
speculoos = "0.9.0" # Fluent assertions
[features]
default = ["billing", "dep:billing_component"]
billing = ["dep:billing_component"]
all_except_billing = []
@@ -1,3 +1,5 @@
#[cfg(feature = "billing")]
use billing_component::stripe::traits::internal_product_to_stripe_lookup::{InternalProductToStripeLookup, StripeProduct, StripeProductLookupError};
use reusable_types::server_environment::ServerEnvironment;
@@ -7,6 +9,8 @@ use crate::configs::plans::plan_list::{DEVELOPMENT_PREMIUM_PLANS_BY_SLUG, PRODUC
#[derive(Clone, Copy)]
pub struct InternalProductToStripeLookupImpl;
#[cfg(feature = "billing")]
impl InternalProductToStripeLookup for InternalProductToStripeLookupImpl {
fn lookup_stripe_product_from_internal_product_key(&self, server_environment: ServerEnvironment, internal_product_key: &str) -> Result<Option<StripeProduct>, StripeProductLookupError> {
let plans_by_slug = match server_environment {
@@ -1,5 +1,6 @@
use actix_web::HttpRequest;
#[cfg(feature = "billing")]
use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge;
use redis_caching::redis_ttl_cache::RedisTtlCache;
use redis_common::redis_cache_keys::RedisCacheKeys;
@@ -19,6 +20,7 @@ impl InternalSessionCachePurgeImpl {
}
}
#[cfg(feature = "billing")]
impl InternalSessionCachePurge for InternalSessionCachePurgeImpl {
fn best_effort_purge_session_cache(&self, http_request: &HttpRequest) {
// TODO: Clear Redis cache of sessions
@@ -1,3 +1,5 @@
#[cfg(feature = "billing")]
use billing_component::stripe::traits::internal_subscription_product_lookup::{InternalProductLookupError, InternalSubscriptionProduct, InternalSubscriptionProductLookup};
use crate::configs::plans::plan_list::{PLANS_BY_STRIPE_PRICE_ID, PLANS_BY_STRIPE_PRODUCT_ID};
@@ -8,6 +10,7 @@ const SUBSCRIPTION_CATEGORY : &str = "fakeyou";
#[derive(Clone, Copy)]
pub struct StripeInternalSubscriptionProductLookupImpl;
#[cfg(feature = "billing")]
impl InternalSubscriptionProductLookup for StripeInternalSubscriptionProductLookupImpl {
fn lookup_internal_product_from_stripe_product_id(&self, stripe_product_id: &str) -> Result<Option<InternalSubscriptionProduct>, InternalProductLookupError> {
Ok(PLANS_BY_STRIPE_PRODUCT_ID.get(stripe_product_id)
@@ -3,6 +3,7 @@ use async_trait::async_trait;
use chrono::Utc;
use log::warn;
#[cfg(feature = "billing")]
use billing_component::stripe::traits::internal_user_lookup::{InternalUserLookup, InternalUserLookupError, SubscriptionKey, UserMetadata};
use users_component::utils::session_checker::SessionChecker;
@@ -25,6 +26,7 @@ impl StripeInternalUserLookupImpl {
}
// NB: Marking async_trait as not needing Sync/Send. Hopefully this doesn't blow up on us.
#[cfg(feature = "billing")]
#[async_trait(?Send)]
impl InternalUserLookup for StripeInternalUserLookupImpl {
async fn lookup_user_from_http_request(&self, http_request: &HttpRequest) -> Result<Option<UserMetadata>, InternalUserLookupError> {
@@ -37,7 +37,7 @@ const DEBUG_HEADER_NAME : &str = "enable-debug-mode";
const ROUTING_TAG_HEADER_NAME : &str = "routing-tag";
#[derive(Deserialize)]
pub struct EnqueueLipsyncAnimationRequest {
pub struct EnqueueFaceAnimationRequest {
uuid_idempotency_token: String,
audio_source: AudioSource,
@@ -117,35 +117,35 @@ pub struct ImageSource {
}
#[derive(Serialize)]
pub struct EnqueueLipsyncAnimationSuccessResponse {
pub struct EnqueueFaceAnimationSuccessResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
#[derive(Debug)]
pub enum EnqueueLipsyncAnimationError {
pub enum EnqueueFaceAnimationError {
BadInput(String),
NotAuthorized,
ServerError,
RateLimited,
}
impl ResponseError for EnqueueLipsyncAnimationError {
impl ResponseError for EnqueueFaceAnimationError {
fn status_code(&self) -> StatusCode {
match *self {
EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST,
EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED,
EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
EnqueueFaceAnimationError::BadInput(_) => StatusCode::BAD_REQUEST,
EnqueueFaceAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED,
EnqueueFaceAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
EnqueueFaceAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(),
EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(),
EnqueueLipsyncAnimationError::ServerError => "server error".to_string(),
EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(),
EnqueueFaceAnimationError::BadInput(reason) => reason.to_string(),
EnqueueFaceAnimationError::NotAuthorized => "unauthorized".to_string(),
EnqueueFaceAnimationError::ServerError => "server error".to_string(),
EnqueueFaceAnimationError::RateLimited => "rate limited".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
@@ -153,16 +153,16 @@ impl ResponseError for EnqueueLipsyncAnimationError {
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl std::fmt::Display for EnqueueLipsyncAnimationError {
impl std::fmt::Display for EnqueueFaceAnimationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
pub async fn enqueue_lipsync_animation_handler(
pub async fn enqueue_face_animation_handler(
http_request: HttpRequest,
request: web::Json<EnqueueLipsyncAnimationRequest>,
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, EnqueueLipsyncAnimationError>
request: web::Json<EnqueueFaceAnimationRequest>,
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, EnqueueFaceAnimationError>
{
let mut maybe_user_token : Option<UserToken> = None;
@@ -171,7 +171,7 @@ pub async fn enqueue_lipsync_animation_handler(
.await
.map_err(|err| {
warn!("MySql pool error: {:?}", err);
EnqueueLipsyncAnimationError::ServerError
EnqueueFaceAnimationError::ServerError
})?;
let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request);
@@ -184,7 +184,7 @@ pub async fn enqueue_lipsync_animation_handler(
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
EnqueueLipsyncAnimationError::ServerError
EnqueueFaceAnimationError::ServerError
})?;
if let Some(user_session) = maybe_user_session.as_ref() {
@@ -215,14 +215,14 @@ pub async fn enqueue_lipsync_animation_handler(
None => &server_state.redis_rate_limiters.logged_out,
Some(ref user) => {
if user.role.is_banned {
return Err(EnqueueLipsyncAnimationError::NotAuthorized);
return Err(EnqueueFaceAnimationError::NotAuthorized);
}
&server_state.redis_rate_limiters.logged_in
},
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
return Err(EnqueueLipsyncAnimationError::RateLimited);
return Err(EnqueueFaceAnimationError::RateLimited);
}
// ==================== LOOK UP MODEL INFO ==================== //
@@ -239,7 +239,7 @@ pub async fn enqueue_lipsync_animation_handler(
} else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token {
LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str())
} else {
return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string()));
return Err(EnqueueFaceAnimationError::BadInput("audio source not fully specified".to_string()));
}
};
@@ -249,7 +249,7 @@ pub async fn enqueue_lipsync_animation_handler(
} else if let Some(ref token) = request.image_source.maybe_media_upload_token {
LipsyncAnimationImageSource::media_upload_token(token.as_str())
} else {
return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string()));
return Err(EnqueueFaceAnimationError::BadInput("image source not fully specified".to_string()));
}
};
@@ -329,7 +329,7 @@ pub async fn enqueue_lipsync_animation_handler(
Ok((job_token, _id)) => job_token,
Err(err) => {
warn!("New generic inference job creation DB error: {:?}", err);
return Err(EnqueueLipsyncAnimationError::ServerError);
return Err(EnqueueFaceAnimationError::ServerError);
}
};
@@ -341,16 +341,16 @@ pub async fn enqueue_lipsync_animation_handler(
.await
.map_err(|e| {
warn!("error publishing event: {:?}", e);
EnqueueLipsyncAnimationError::ServerError
EnqueueFaceAnimationError::ServerError
})?;
let response: EnqueueLipsyncAnimationSuccessResponse = EnqueueLipsyncAnimationSuccessResponse {
let response: EnqueueFaceAnimationSuccessResponse = EnqueueFaceAnimationSuccessResponse {
success: true,
inference_job_token: job_token,
};
let body = serde_json::to_string(&response)
.map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?;
.map_err(|_e| EnqueueFaceAnimationError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -1 +1 @@
pub mod enqueue_lipsync_animation;
pub mod enqueue_face_animation;
+113 -67
View File
@@ -46,11 +46,6 @@ use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoint_filte
use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::disabled_endpoints::DisabledEndpoints;
use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::exact_match_disabled_endpoints::ExactMatchDisabledEndpoints;
use actix_helpers::middleware::disabled_endpoint_filter::disabled_endpoints::prefix_disabled_endpoints::PrefixDisabledEndpoints;
use billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets};
use billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup;
use billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge;
use billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup;
use billing_component::stripe::traits::internal_user_lookup::InternalUserLookup;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use config::common_env::CommonEnv;
@@ -71,10 +66,19 @@ use users_component::cookies::anonymous_visitor_tracking::avt_cookie_manager::Av
use users_component::cookies::session::session_cookie_manager::SessionCookieManager;
use users_component::utils::session_checker::SessionChecker;
use crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl;
use crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl;
use crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl;
use crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl;
#[cfg(feature = "billing")]
use
{
crate::billing::internal_product_to_stripe_lookup_impl::InternalProductToStripeLookupImpl,
crate::billing::internal_session_cache_purge_impl::InternalSessionCachePurgeImpl,
crate::billing::stripe_internal_subscription_product_lookup_impl::StripeInternalSubscriptionProductLookupImpl,
crate::billing::stripe_internal_user_lookup_impl::StripeInternalUserLookupImpl,
billing_component::stripe::stripe_config::{FullUrlOrPath, StripeCheckoutConfigs, StripeConfig, StripeCustomerPortalConfigs, StripeSecrets},
billing_component::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup,
billing_component::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge,
billing_component::stripe::traits::internal_subscription_product_lookup::InternalSubscriptionProductLookup,
billing_component::stripe::traits::internal_user_lookup::InternalUserLookup,
};
use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet};
use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter;
use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter;
@@ -382,6 +386,7 @@ async fn main() -> AnyhowResult<()> {
});
#[cfg(feature = "billing")]
let stripe_configs = StripeConfig {
checkout: StripeCheckoutConfigs {
success_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_SUCCESS_URL_PATH")?),
@@ -398,6 +403,9 @@ async fn main() -> AnyhowResult<()> {
},
};
#[cfg(not(feature = "billing"))]
let stripe_configs = ();
let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?)
.ok_or(anyhow!("invalid server environment"))?;
@@ -424,10 +432,6 @@ async fn main() -> AnyhowResult<()> {
let third_party_url_redirector = ThirdPartyUrlRedirector::new(server_environment);
let stripe_client = {
let api_secret = stripe_configs.secrets.secret_key.clone();
stripe::Client::new(api_secret)
};
// NB: Docker creates this file within container builds.
let build_sha = std::fs::read_to_string("/GIT_SHA")
@@ -455,7 +459,6 @@ async fn main() -> AnyhowResult<()> {
},
stripe: StripeSettings {
config: stripe_configs,
client: stripe_client,
},
hostname: server_hostname,
server_environment,
@@ -636,62 +639,105 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()>
//let log_format = "[%{HOSTNAME}e] %a \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T";
let log_format = "[%{HOSTNAME}e] IP=[%{X-Forwarded-For}i] \"%r\" %s %b \"%{Referer}i\" \"%{User-Agent}i\" %T";
HttpServer::new(move || {
// NB: Safe to clone due to internal arc
let ip_ban_list = server_state_arc.ip_ban_list.clone();
let cidr_ban_set= server_state_arc.cidr_ban_set.clone();
#[cfg(feature = "billing")]
{
let server = HttpServer::new(move || {
// NB: Safe to clone due to internal arc
let ip_ban_list = server_state_arc.ip_ban_list.clone();
let cidr_ban_set= server_state_arc.cidr_ban_set.clone();
// NB: Dynamic dispatch needs to be wrapped with Arc.
let product_lookup : Arc<dyn InternalSubscriptionProductLookup> = Arc::new(StripeInternalSubscriptionProductLookupImpl {});
let stripe_lookup : Arc<dyn InternalProductToStripeLookup> = Arc::new(InternalProductToStripeLookupImpl{});
let user_lookup : Arc<dyn InternalUserLookup> = Arc::new(StripeInternalUserLookupImpl::new(
server_state_arc.session_checker.clone(),
server_state_arc.mysql_pool.clone(),
));
let session_cache_purge : Arc<dyn InternalSessionCachePurge> = Arc::new(InternalSessionCachePurgeImpl::new(
server_state_arc.session_checker.clone(),
server_state_arc.redis_ttl_cache.clone(),
));
// NB: Dynamic dispatch needs to be wrapped with Arc.
let product_lookup : Arc<dyn InternalSubscriptionProductLookup> = Arc::new(StripeInternalSubscriptionProductLookupImpl {});
let stripe_lookup : Arc<dyn InternalProductToStripeLookup> = Arc::new(InternalProductToStripeLookupImpl{});
let user_lookup : Arc<dyn InternalUserLookup> = Arc::new(StripeInternalUserLookupImpl::new(
server_state_arc.session_checker.clone(),
server_state_arc.mysql_pool.clone(),
));
let session_cache_purge : Arc<dyn InternalSessionCachePurge> = Arc::new(InternalSessionCachePurgeImpl::new(
server_state_arc.session_checker.clone(),
server_state_arc.redis_ttl_cache.clone(),
));
// NB: app_data being clone()'d below should all be safe (dependencies included)
let app = App::new()
.app_data(web::Data::new(server_state_arc.firehose_publisher.clone()))
.app_data(web::Data::new(server_state_arc.mysql_pool.clone()))
.app_data(web::Data::new(server_state_arc.redis_pool.clone()))
.app_data(web::Data::new(server_state_arc.redis_ttl_cache.clone()))
.app_data(web::Data::new(server_state_arc.session_checker.clone()))
.app_data(web::Data::new(server_state_arc.avt_cookie_manager.clone()))
.app_data(web::Data::new(server_state_arc.session_cookie_manager.clone()))
.app_data(web::Data::new(server_state_arc.stripe.clone().config.clone()))
.app_data(web::Data::new(server_state_arc.stripe.clone().client.clone()))
.app_data(web::Data::new(server_state_arc.third_party_url_redirector.clone()))
.app_data(web::Data::new(server_state_arc.email_sender.clone()))
.app_data(web::Data::new(old_server_environment))
.app_data(web::Data::new(new_server_environment))
.app_data(web::Data::from(product_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(web::Data::from(stripe_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(web::Data::from(user_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(web::Data::from(session_cache_purge)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(server_state_arc.clone())
.wrap(build_cors_config(old_server_environment))
.wrap(DefaultHeaders::new()
.header("X-Backend-Hostname", &hostname)
.header("X-Build-Sha", server_state_arc.server_info.build_sha.clone()))
.wrap(PushbackFilter::new(&server_state_arc.flags.clone()))
.wrap(DisabledEndpointFilter::new(disabled_endpoints.clone()))
.wrap(BannedIpFilter::new(ip_ban_list))
.wrap(BannedCidrFilter::new(cidr_ban_set))
.wrap(Logger::new(&log_format)
.exclude("/liveness")
.exclude("/readiness"))
.wrap(middleware::Compress::default());
// NB: app_data being clone()'d below should all be safe (dependencies included)
let app = App::new()
.app_data(web::Data::new(server_state_arc.firehose_publisher.clone()))
.app_data(web::Data::new(server_state_arc.mysql_pool.clone()))
.app_data(web::Data::new(server_state_arc.redis_pool.clone()))
.app_data(web::Data::new(server_state_arc.redis_ttl_cache.clone()))
.app_data(web::Data::new(server_state_arc.session_checker.clone()))
.app_data(web::Data::new(server_state_arc.avt_cookie_manager.clone()))
.app_data(web::Data::new(server_state_arc.session_cookie_manager.clone()))
.app_data(web::Data::new(server_state_arc.stripe.clone().config.clone()))
.app_data(web::Data::new(server_state_arc.third_party_url_redirector.clone()))
.app_data(web::Data::new(server_state_arc.email_sender.clone()))
.app_data(web::Data::new(old_server_environment))
.app_data(web::Data::new(new_server_environment))
.app_data(web::Data::from(product_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(web::Data::from(stripe_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(web::Data::from(user_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(web::Data::from(session_cache_purge)) // NB: Data::from(Arc<T>) for dynamic dispatch
.app_data(server_state_arc.clone())
.wrap(build_cors_config(old_server_environment))
.wrap(DefaultHeaders::new()
.header("X-Backend-Hostname", &hostname)
.header("X-Build-Sha", server_state_arc.server_info.build_sha.clone()))
.wrap(PushbackFilter::new(&server_state_arc.flags.clone()))
.wrap(DisabledEndpointFilter::new(disabled_endpoints.clone()))
.wrap(BannedIpFilter::new(ip_ban_list))
.wrap(BannedCidrFilter::new(cidr_ban_set))
.wrap(Logger::new(&log_format)
.exclude("/liveness")
.exclude("/readiness"))
.wrap(middleware::Compress::default());
add_routes(app, old_server_environment)
})
.bind(bind_address)?
.workers(num_workers)
.run()
.await?;
crate::routes::add_routes(app, old_server_environment)
});
server.bind(bind_address)?
.workers(num_workers)
.run()
.await?;
}
#[cfg(not(feature = "billing"))] {
let server = HttpServer::new(move || {
// NB: Safe to clone due to internal arc
let ip_ban_list = server_state_arc.ip_ban_list.clone();
let cidr_ban_set= server_state_arc.cidr_ban_set.clone();
// NB: app_data being clone()'d below should all be safe (dependencies included)
let app = App::new()
.app_data(web::Data::new(server_state_arc.firehose_publisher.clone()))
.app_data(web::Data::new(server_state_arc.mysql_pool.clone()))
.app_data(web::Data::new(server_state_arc.redis_pool.clone()))
.app_data(web::Data::new(server_state_arc.redis_ttl_cache.clone()))
.app_data(web::Data::new(server_state_arc.session_checker.clone()))
.app_data(web::Data::new(server_state_arc.avt_cookie_manager.clone()))
.app_data(web::Data::new(server_state_arc.session_cookie_manager.clone()))
.app_data(web::Data::new(server_state_arc.third_party_url_redirector.clone()))
.app_data(web::Data::new(server_state_arc.email_sender.clone()))
.app_data(web::Data::new(old_server_environment))
.app_data(web::Data::new(new_server_environment))
.app_data(server_state_arc.clone())
.wrap(build_cors_config(old_server_environment))
.wrap(DefaultHeaders::new()
.header("X-Backend-Hostname", &hostname)
.header("X-Build-Sha", server_state_arc.server_info.build_sha.clone()))
.wrap(PushbackFilter::new(&server_state_arc.flags.clone()))
.wrap(DisabledEndpointFilter::new(disabled_endpoints.clone()))
.wrap(BannedIpFilter::new(ip_ban_list))
.wrap(BannedCidrFilter::new(cidr_ban_set))
.wrap(Logger::new(&log_format)
.exclude("/liveness")
.exclude("/readiness"))
.wrap(middleware::Compress::default());
add_routes(app, old_server_environment)
});
server.bind(bind_address)?
.workers(num_workers)
.run()
.await?;
}
Ok(())
}
@@ -5,13 +5,14 @@ use actix_web::dev::{ServiceRequest, ServiceResponse};
use actix_web::error::Error;
use actix_helpers::route_builder::RouteBuilder;
#[cfg(feature = "billing")]
use billing_component::default_routes::add_suggested_stripe_billing_routes;
use reusable_types::server_environment::ServerEnvironment;
use users_component::default_routes::add_suggested_api_v1_account_creation_and_session_routes;
use users_component::endpoints::edit_profile_handler::edit_profile_handler;
use users_component::endpoints::get_profile_handler::get_profile_handler;
use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler;
use crate::http_server::endpoints::animation::enqueue_face_animation::enqueue_face_animation_handler;
use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler;
use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler;
use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler;
@@ -215,7 +216,7 @@ pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) ->
// ==================== Animations ====================
let mut app = RouteBuilder::from_app(app)
.add_post("/v1/animation/face_animation/create", enqueue_lipsync_animation_handler)
.add_post("/v1/animation/face_animation/create", enqueue_face_animation_handler)
.into_app();
// ==================== "Generic" Inference ====================
@@ -228,15 +229,20 @@ pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) ->
// ==================== Stats ====================
let mut app = RouteBuilder::from_app(app)
.add_get("/v1/stats/queues", get_unified_queue_stats_handler)
.into_app();
// ==================== COMPONENTS ====================
app = add_suggested_api_v1_account_creation_and_session_routes(app); // /create_account, /session, /login, /logout
app = add_suggested_stripe_billing_routes(app); // /stripe, billing, webhooks, etc.
#[cfg(feature = "billing")]
let mut app = add_suggested_stripe_billing_routes(app); // /stripe, billing, webhooks, etc.
#[cfg(not(feature = "billing"))]
let mut app = app;
// ==================== SERVICE ====================
app.service(
web::resource("/_status")
@@ -4,6 +4,7 @@ use sqlx::MySqlPool;
use actix_helpers::middleware::banned_cidr_filter::banned_cidr_set::BannedCidrSet;
use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBanList;
#[cfg(feature = "billing")]
use billing_component::stripe::stripe_config::StripeConfig;
use cloud_storage::bucket_client::BucketClient;
use email_sender::smtp_email_sender::SmtpEmailSender;
@@ -197,10 +198,16 @@ pub struct EphemeralInMemoryCaches {
pub leaderboard: SingleItemTtlCache<LeaderboardInfo>,
}
#[cfg(feature = "billing")]
#[derive(Clone)]
pub struct StripeSettings {
pub config: StripeConfig,
pub client: stripe::Client,
}
#[cfg(not(feature = "billing"))]
#[derive(Clone)]
pub struct StripeSettings {
pub config: ()
}
/// Flags set at service startup
@@ -24,6 +24,9 @@ mysql_queries = { path = "../../../lib/mysql_queries" }
redis_common = { path = "../../../lib/redis_common" }
twitch_common = { path = "../../../lib/api_clients/twitch_common" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
base64 = "0.21.0"
@@ -89,9 +92,6 @@ serde_derive = "1.0.125"
serde_json = "1.0.108"
toml = "0.5.8"
# Temporary (just to ask for oauth credentials during dev)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
[build-dependencies]
# NB: Only using diesel for migrations support
diesel = { version = "2.1.3", features = ["mysql"] }
+3
View File
@@ -23,6 +23,9 @@ futures = "0.1.29"
redis = "0.13.0"
time = "0.1.42"
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
tokio = "0.1.22"
version-sync = "0.8.1"
+2
View File
@@ -21,6 +21,8 @@ flate2 = { version = "1.0.16", optional = true }
futures = { version = "0.3.1", optional = true }
tokio = { version = "1.1.0", features = ["macros"], optional = true }
uuid = { version = "0.8.1", features = ["v4"] }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
[dev-dependencies]
+6
View File
@@ -13,6 +13,7 @@ cargo test \
--exclude billing_component \
--exclude fakeyou_client \
--exclude inference-job \
--exclude mysql_queries \
--exclude sqlite_queries
# Single broken tests
@@ -22,4 +23,9 @@ cargo test \
--skip util::audiowmark::tests::test_audiowmark \
--skip util::common_commands::audiowmark::tests::test_audiowmark
cargo test \
--package mysql_queries \
-- \
--skip queries::model_weights::tests
echo 'Tests passed.'