Compare commits

...
224 changed files with 5402 additions and 11933 deletions
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO debug_logs\nSET\n event_token = ?,\n debug_log_type = ?,\n maybe_log_level = ?,\n maybe_creator_user_token = ?,\n maybe_ip_address = ?,\n maybe_url = ?,\n message = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 7
},
"nullable": []
},
"hash": "32a8d1cfec89ba2056463a33ddc501a8a99283c7c00982a7fa79f7ff382c9b22"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nUPDATE tts_results\nSET\n mod_deleted_at = CURRENT_TIMESTAMP,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "376e36b1357952599866262c7356ba35f15ccdfb338d8ccdbdc06ccd69331fc3"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nUPDATE tts_results\nSET\n mod_deleted_at = NULL,\n maybe_mod_user_token = ?\nWHERE\n token = ?\nLIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO voice_clone_requests\nSET\n token = ?,\n uuid_idempotency_token = ?,\n\n maybe_user_token = ?,\n email_address = ?,\n discord_username = ?,\n\n is_for_private_use = ?,\n is_for_public_use = ?,\n\n is_for_studio = ?,\n is_for_twitch_tts = ?,\n is_for_api_use = ?,\n is_for_music = ?,\n is_for_games = ?,\n is_for_other = ?,\n optional_notes_on_use = ?,\n\n is_own_voice = ?,\n is_third_party_voice = ?,\n optional_notes_on_subject = ?,\n\n has_clean_audio_recordings = ?,\n has_good_microphone = ?,\n\n optional_questions = ?,\n optional_extra_comments = ?,\n\n ip_address_creation = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 22
},
"nullable": []
},
"hash": "4cea10944b8869419ecdfea30c58f49a87c090b9083078043177c33737ecb949"
}
@@ -1,36 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n COUNT(distinct token) as record_count,\n NOW() as present_time\nFROM\n(\n SELECT token\n FROM tts_inference_jobs\n WHERE status = \"started\"\n AND created_at > (CURDATE() - INTERVAL 5 MINUTE)\nUNION\n SELECT token\n FROM tts_inference_jobs\n WHERE status IN (\"pending\", \"attempt_failed\")\n) as t\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "record_count",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | BINARY",
"char_set": 63,
"max_size": 21
}
},
{
"ordinal": 1,
"name": "present_time",
"type_info": {
"type": "Datetime",
"flags": "NOT_NULL | BINARY",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 0
},
"nullable": [
false,
false
]
},
"hash": "6098826a8ada75442be5b7ac1d0199694e01c5959666f59986cd8b2ed6f51882"
}
@@ -1,47 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n job_type as queue_type,\n count(*) as pending_job_count,\n NOW() as present_time\n FROM (\n SELECT\n token,\n job_type\n FROM generic_inference_jobs\n WHERE status IN (\"pending\", \"attempt_failed\")\n AND job_type IS NOT NULL\n UNION\n SELECT\n token,\n job_type\n FROM generic_inference_jobs\n WHERE status IN (\"started\")\n AND created_at > (CURDATE() - INTERVAL 15 MINUTE)\n AND job_type IS NOT NULL\n) as generic_inner\nGROUP BY queue_type\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "queue_type",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "pending_job_count",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | BINARY",
"char_set": 63,
"max_size": 21
}
},
{
"ordinal": 2,
"name": "present_time",
"type_info": {
"type": "Datetime",
"flags": "NOT_NULL | BINARY",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 0
},
"nullable": [
true,
false,
false
]
},
"hash": "73dc054488b2fa3efc923d3a90914a1450b73e99a377562484a0498ccce1676f"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO tts_model_upload_jobs\nSET\n token = ?,\n uuid_idempotency_token = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n title = ?,\n tts_model_type = ?,\n download_url = ?,\n status = \"pending\"\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 8
},
"nullable": []
},
"hash": "886861e91c7e8af53eb856c949389a172b8d9e079f1c2070d115423907c47c50"
}
@@ -0,0 +1,102 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n users.token as `user_token: UserToken`,\n users.username,\n users.display_name,\n users.email_address,\n users.user_role_slug,\n users.is_banned,\n user_roles.can_ban_users,\n api_keys.token as `api_key_token: ApiKeyToken`\nFROM api_keys\nINNER JOIN users\n ON users.token = api_keys.owner_user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE api_keys.api_key = ?\n AND api_keys.maybe_deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\nLIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "user_token: UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 2,
"name": "display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 3,
"name": "email_address",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 4,
"name": "user_role_slug",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 5,
"name": "is_banned",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 6,
"name": "can_ban_users",
"type_info": {
"type": "Tiny",
"flags": "",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 7,
"name": "api_key_token: ApiKeyToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
false,
false,
false,
false,
false,
true,
false
]
},
"hash": "9188e235caa439d50628333c2f4722b9f1a8f575c192d4d36a2039709c412ac0"
}
@@ -1,256 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_private_use,\n is_for_public_use,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_own_voice,\n is_third_party_voice,\n optional_notes_on_subject,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n token = ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "maybe_user_token",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 2,
"name": "email_address",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 3,
"name": "discord_username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 4,
"name": "is_for_private_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 5,
"name": "is_for_public_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 6,
"name": "is_for_studio",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 7,
"name": "is_for_twitch_tts",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 8,
"name": "is_for_api_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 9,
"name": "is_for_music",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 10,
"name": "is_for_games",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 11,
"name": "is_for_other",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 12,
"name": "optional_notes_on_use",
"type_info": {
"type": "Blob",
"flags": "BLOB | BINARY",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 13,
"name": "is_own_voice",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 14,
"name": "is_third_party_voice",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 15,
"name": "optional_notes_on_subject",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 16,
"name": "has_clean_audio_recordings",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 17,
"name": "has_good_microphone",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 18,
"name": "optional_questions",
"type_info": {
"type": "Blob",
"flags": "BLOB | BINARY",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 19,
"name": "optional_extra_comments",
"type_info": {
"type": "Blob",
"flags": "BLOB | BINARY",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 20,
"name": "created_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 21,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
true,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
true,
false,
false,
true,
true,
false,
false
]
},
"hash": "922a7dbcfa24a2c5901d0453c229b066b3b4eea2c73fa533651bb27365f17be0"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nUPDATE tts_results\nSET\n user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n token = ?\nLIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1"
}
@@ -1,135 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n jobs.token as job_token,\n\n jobs.status,\n jobs.attempt_count,\n jobs.on_success_result_token as maybe_result_token,\n results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,\n\n jobs.model_token,\n tts.tts_model_type,\n tts.title,\n\n jobs.raw_inference_text,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM tts_inference_jobs as jobs\nJOIN tts_models as tts\n ON tts.token = jobs.model_token\nLEFT OUTER JOIN tts_results as results\n ON jobs.on_success_result_token = results.token\n\nWHERE jobs.token = ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "job_token",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "status",
"type_info": {
"type": "String",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 2,
"name": "attempt_count",
"type_info": {
"type": "Long",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 3
}
},
{
"ordinal": 3,
"name": "maybe_result_token",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 4,
"name": "maybe_public_bucket_wav_audio_path",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 5,
"name": "model_token",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 6,
"name": "tts_model_type",
"type_info": {
"type": "String",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
"char_set": 224,
"max_size": 60
}
},
{
"ordinal": 7,
"name": "title",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 8,
"name": "raw_inference_text",
"type_info": {
"type": "Blob",
"flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 262140
}
},
{
"ordinal": 9,
"name": "created_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 10,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
false,
false,
true,
true,
false,
false,
false,
false,
false,
false
]
},
"hash": "a2b586dd2120f29dd26281626f7381e292c821f8e826b9bc584b92ffa45035b2"
}
@@ -1,80 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n id as `id: u64`,\n event_token as `event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,\n debug_log_type as `debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType`,\n maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n message,\n created_at\nFROM debug_logs\nWHERE event_token = ?\nORDER BY id ASC\nLIMIT ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id: u64",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | PRIMARY_KEY | UNSIGNED | AUTO_INCREMENT",
"char_set": 63,
"max_size": 20
}
},
{
"ordinal": 1,
"name": "event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 2,
"name": "debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 96
}
},
{
"ordinal": 3,
"name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 4,
"name": "message",
"type_info": {
"type": "Blob",
"flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 5,
"name": "created_at",
"type_info": {
"type": "Datetime",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 2
},
"nullable": [
false,
false,
false,
true,
false,
false
]
},
"hash": "a62b88fec40a11952cc199383090ea338f97091e8bfc6b68aad67416685889af"
}
@@ -1,69 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n users.username,\n users.display_name,\n users.email_gravatar_hash as gravatar_hash,\n creator_user_token,\n count(*) as uploaded_count\nFROM tts_models\nJOIN users\nON\n users.token = creator_user_token\nWHERE\n users.is_banned IS FALSE\n AND users.token NOT IN (\"U:DRB1SKEMKHT1J\")\n AND tts_models.user_deleted_at IS NULL\n AND tts_models.mod_deleted_at IS NULL\nGROUP BY creator_user_token\nORDER BY uploaded_count desc\nLIMIT 25;\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 1,
"name": "display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 2,
"name": "gravatar_hash",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 3,
"name": "creator_user_token",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 4,
"name": "uploaded_count",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | BINARY",
"char_set": 63,
"max_size": 21
}
}
],
"parameters": {
"Right": 0
},
"nullable": [
false,
false,
false,
false,
false
]
},
"hash": "a82fa4eb36528ebf52af2585cdda203476a4ba3645837c792830474fca2eda4f"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO debug_logs\nSET\n event_token = ?,\n debug_log_type = ?,\n maybe_creator_user_token = ?,\n message = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 4
},
"nullable": []
},
"hash": "a9745ecf60b564ab096ee1dbe124221b5f288a4ae38e788e19bbd359db0b580f"
}
@@ -0,0 +1,113 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n e.maybe_user_token AS `user_token!: UserToken`,\n u.username AS `username!`,\n u.display_name AS `display_name!`,\n u.email_gravatar_hash AS `email_gravatar_hash!`,\n CAST(SUM(CASE WHEN e.amount_usd_cents > 0 THEN e.amount_usd_cents ELSE 0 END) AS UNSIGNED) AS `gross_spend_usd_cents!: u64`,\n CAST(SUM(CASE WHEN e.amount_usd_cents < 0 THEN -e.amount_usd_cents ELSE 0 END) AS UNSIGNED) AS `refund_usd_cents!: u64`,\n CAST(SUM(e.amount_usd_cents) AS SIGNED) AS `net_spend_usd_cents!: i64`,\n CAST(SUM(CASE WHEN e.amount_usd_cents > 0 THEN 1 ELSE 0 END) AS UNSIGNED) AS `payment_count!: u64`,\n CAST(COALESCE(SUM(e.maybe_credits_granted), 0) AS UNSIGNED) AS `credits_granted!: u64`\nFROM user_spend_events e\nJOIN users u ON u.token = e.maybe_user_token\nWHERE e.maybe_user_token IS NOT NULL\n AND e.is_production = TRUE\n AND e.payment_occurred_at >= ?\n AND (? IS NULL OR e.payments_namespace = ?)\nGROUP BY e.maybe_user_token, u.username, u.display_name, u.email_gravatar_hash\nORDER BY SUM(e.amount_usd_cents) DESC, e.maybe_user_token ASC\nLIMIT ? OFFSET ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "user_token!: UserToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "username!",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 2,
"name": "display_name!",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 3,
"name": "email_gravatar_hash!",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 4,
"name": "gross_spend_usd_cents!: u64",
"type_info": {
"type": "LongLong",
"flags": "UNSIGNED | BINARY",
"char_set": 63,
"max_size": 21
}
},
{
"ordinal": 5,
"name": "refund_usd_cents!: u64",
"type_info": {
"type": "LongLong",
"flags": "UNSIGNED | BINARY",
"char_set": 63,
"max_size": 21
}
},
{
"ordinal": 6,
"name": "net_spend_usd_cents!: i64",
"type_info": {
"type": "LongLong",
"flags": "BINARY",
"char_set": 63,
"max_size": 21
}
},
{
"ordinal": 7,
"name": "payment_count!: u64",
"type_info": {
"type": "LongLong",
"flags": "UNSIGNED | BINARY",
"char_set": 63,
"max_size": 21
}
},
{
"ordinal": 8,
"name": "credits_granted!: u64",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | UNSIGNED | BINARY",
"char_set": 63,
"max_size": 21
}
}
],
"parameters": {
"Right": 5
},
"nullable": [
true,
false,
false,
false,
true,
true,
true,
true,
false
]
},
"hash": "c2ee41eb6bc1a6a551411998155423142411010f298f0815eacc7d9c3138bdd8"
}
@@ -0,0 +1,91 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n users.token as `user_token: UserToken`,\n users.username,\n users.display_name,\n users.email_address,\n users.user_role_slug,\n users.is_banned,\n user_roles.can_ban_users\nFROM user_sessions\nINNER JOIN users\n ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n AND user_sessions.deleted_at IS NULL\n AND users.user_deleted_at IS NULL\n AND users.mod_deleted_at IS NULL\nLIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "user_token: UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 2,
"name": "display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 3,
"name": "email_address",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 4,
"name": "user_role_slug",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 5,
"name": "is_banned",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 6,
"name": "can_ban_users",
"type_info": {
"type": "Tiny",
"flags": "",
"char_set": 63,
"max_size": 1
}
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
false,
false,
false,
false,
false,
true
]
},
"hash": "c77126adb36f9e2f0248720eb4d8c1cd9d14719f64a02678d8021e26addd2c18"
}
@@ -0,0 +1,146 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n d.id as `id: u64`,\n d.event_token as `event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,\n d.debug_log_type as `debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType`,\n d.maybe_log_level as `maybe_log_level: enums::by_table::debug_logs::debug_log_level::DebugLogLevel`,\n d.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n d.maybe_ip_address,\n d.maybe_url,\n d.message,\n d.created_at,\n u.display_name as `maybe_user_display_name?`,\n u.username as `maybe_user_username?`,\n u.email_gravatar_hash as `maybe_user_gravatar_hash?`\nFROM debug_logs d\nLEFT OUTER JOIN users u\n ON u.token = d.maybe_creator_user_token\nWHERE d.maybe_creator_user_token = ?\n AND d.id < ?\nORDER BY d.id DESC\nLIMIT ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id: u64",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | PRIMARY_KEY | UNSIGNED | AUTO_INCREMENT",
"char_set": 63,
"max_size": 20
}
},
{
"ordinal": 1,
"name": "event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 2,
"name": "debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 96
}
},
{
"ordinal": 3,
"name": "maybe_log_level: enums::by_table::debug_logs::debug_log_level::DebugLogLevel",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 4,
"name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 5,
"name": "maybe_ip_address",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 160
}
},
{
"ordinal": 6,
"name": "maybe_url",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 7,
"name": "message",
"type_info": {
"type": "Blob",
"flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 8,
"name": "created_at",
"type_info": {
"type": "Datetime",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 9,
"name": "maybe_user_display_name?",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 10,
"name": "maybe_user_username?",
"type_info": {
"type": "VarString",
"flags": "UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 11,
"name": "maybe_user_gravatar_hash?",
"type_info": {
"type": "String",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
}
],
"parameters": {
"Right": 3
},
"nullable": [
false,
false,
false,
true,
true,
true,
true,
false,
false,
true,
true,
true
]
},
"hash": "d1ade7fcfd3efac1cd3163c392110327bb73edb74fa0d9693fee5feddb8a3223"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nUPDATE tts_results\nSET\n user_deleted_at = NULL\nWHERE\n token = ?\nLIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 1
},
"nullable": []
},
"hash": "dbce51eb1c80dded902c9509e50713483c7131f115e8ec10646e99e88110278b"
}
@@ -1,256 +0,0 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n token,\n\n maybe_user_token,\n email_address,\n discord_username,\n\n is_for_private_use,\n is_for_public_use,\n\n is_for_studio,\n is_for_twitch_tts,\n is_for_api_use,\n is_for_music,\n is_for_games,\n is_for_other,\n optional_notes_on_use,\n\n is_own_voice,\n is_third_party_voice,\n optional_notes_on_subject,\n\n has_clean_audio_recordings,\n has_good_microphone,\n\n optional_questions,\n optional_extra_comments,\n\n created_at,\n updated_at\nFROM voice_clone_requests\nWHERE\n maybe_user_token = ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "token",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 1,
"name": "maybe_user_token",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 2,
"name": "email_address",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 3,
"name": "discord_username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 4,
"name": "is_for_private_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 5,
"name": "is_for_public_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 6,
"name": "is_for_studio",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 7,
"name": "is_for_twitch_tts",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 8,
"name": "is_for_api_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 9,
"name": "is_for_music",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 10,
"name": "is_for_games",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 11,
"name": "is_for_other",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 12,
"name": "optional_notes_on_use",
"type_info": {
"type": "Blob",
"flags": "BLOB | BINARY",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 13,
"name": "is_own_voice",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 14,
"name": "is_third_party_voice",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 15,
"name": "optional_notes_on_subject",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 16,
"name": "has_clean_audio_recordings",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 17,
"name": "has_good_microphone",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 18,
"name": "optional_questions",
"type_info": {
"type": "Blob",
"flags": "BLOB | BINARY",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 19,
"name": "optional_extra_comments",
"type_info": {
"type": "Blob",
"flags": "BLOB | BINARY",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 20,
"name": "created_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 21,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 1
},
"nullable": [
false,
true,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
true,
false,
false,
true,
true,
false,
false
]
},
"hash": "e5ef2bda28221a570da6dcfa73e76cae9910a0b4ee53005a87176ecf1be73dd7"
}
@@ -0,0 +1,146 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n d.id as `id: u64`,\n d.event_token as `event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,\n d.debug_log_type as `debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType`,\n d.maybe_log_level as `maybe_log_level: enums::by_table::debug_logs::debug_log_level::DebugLogLevel`,\n d.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n d.maybe_ip_address,\n d.maybe_url,\n d.message,\n d.created_at,\n u.display_name as `maybe_user_display_name?`,\n u.username as `maybe_user_username?`,\n u.email_gravatar_hash as `maybe_user_gravatar_hash?`\nFROM debug_logs d\nLEFT OUTER JOIN users u\n ON u.token = d.maybe_creator_user_token\nWHERE d.event_token = ?\nORDER BY d.id ASC\nLIMIT ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id: u64",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | PRIMARY_KEY | UNSIGNED | AUTO_INCREMENT",
"char_set": 63,
"max_size": 20
}
},
{
"ordinal": 1,
"name": "event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 2,
"name": "debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 96
}
},
{
"ordinal": 3,
"name": "maybe_log_level: enums::by_table::debug_logs::debug_log_level::DebugLogLevel",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 4,
"name": "maybe_creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 5,
"name": "maybe_ip_address",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 160
}
},
{
"ordinal": 6,
"name": "maybe_url",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 7,
"name": "message",
"type_info": {
"type": "Blob",
"flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 67108860
}
},
{
"ordinal": 8,
"name": "created_at",
"type_info": {
"type": "Datetime",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 9,
"name": "maybe_user_display_name?",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 10,
"name": "maybe_user_username?",
"type_info": {
"type": "VarString",
"flags": "UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 11,
"name": "maybe_user_gravatar_hash?",
"type_info": {
"type": "String",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
}
],
"parameters": {
"Right": 2
},
"nullable": [
false,
false,
false,
true,
true,
true,
true,
false,
false,
true,
true,
true
]
},
"hash": "e9a149ffc963a59019fdd69bd53128fad946856d432108bf9b923dfeaf2442c5"
}
Generated
+15
View File
@@ -3621,6 +3621,7 @@ dependencies = [
"env_logger 0.11.9",
"log",
"storyteller_root",
"trace_id",
"workspace-build-acceleration",
]
@@ -6766,6 +6767,7 @@ version = "0.0.1"
dependencies = [
"env_logger 0.11.9",
"log",
"trace_id",
"workspace-build-acceleration",
]
@@ -8286,6 +8288,7 @@ dependencies = [
"rootly_client",
"sha2 0.11.0-rc.5",
"tokio",
"trace_id",
"workspace-build-acceleration",
]
@@ -11678,6 +11681,7 @@ dependencies = [
"tokens",
"tokio",
"toml 1.0.7+spec-1.1.0",
"trace_id",
"tts_common",
"url",
"url_config",
@@ -13316,6 +13320,17 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "trace_id"
version = "0.0.1"
dependencies = [
"rand 0.8.5",
"serde",
"serde_derive",
"tokio",
"workspace-build-acceleration",
]
[[package]]
name = "tracing"
version = "0.1.41"
+2
View File
@@ -130,6 +130,7 @@ members = [
"crates/schema/public/enums",
"crates/schema/public/reusable_types", # TODO(bt, 2022-12-21): Deprecate
"crates/schema/public/tokens",
"crates/schema/trace_id",
# Services: API servers, jobs, workers, etc.
"crates/service/job/gmicloud_job",
@@ -293,6 +294,7 @@ subprocess_common = { path = "crates/lib/deprecated/subprocess_common" }
test_data = { path = "crates/testing/test_data" }
test_utils = { path = "crates/testing/test_utils" }
tokens = { path = "crates/schema/public/tokens" }
trace_id = { path = "crates/schema/trace_id" }
url_utils = { path = "crates/lib/url_utils" }
users = { path = "crates/lib/users" }
uuid_utils = { path = "crates/lib/uuid_utils" }
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
DROP COLUMN maybe_log_level;
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
ADD COLUMN maybe_log_level VARCHAR(16) DEFAULT NULL;
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
DROP INDEX index_maybe_log_level;
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
ADD INDEX index_maybe_log_level (maybe_log_level);
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
DROP COLUMN maybe_ip_address;
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
ADD COLUMN maybe_ip_address VARCHAR(40) DEFAULT NULL;
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
DROP COLUMN maybe_url;
@@ -0,0 +1,6 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE debug_logs
ADD COLUMN maybe_url VARCHAR(255) DEFAULT NULL;
@@ -0,0 +1,7 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE media_files
DROP INDEX idx_creator_created_at,
ALGORITHM=INPLACE, LOCK=NONE;
@@ -0,0 +1,17 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
-- Composite index for the library / user profile list queries:
-- WHERE maybe_creator_user_token = ? ... ORDER BY created_at DESC LIMIT n
--
-- Without this, the optimizer picks an index-merge intersection against huge
-- low-selectivity indexes (e.g. index_user_deleted_at, ~4.75M NULL entries),
-- scanning millions of index records per page load.
--
-- NB: ADD INDEX is online DDL (ALGORITHM=INPLACE, LOCK=NONE) — reads and writes
-- continue during the build. Expect a few minutes of elevated I/O on a table
-- this size (~8.5M rows).
ALTER TABLE media_files
ADD INDEX idx_creator_created_at (maybe_creator_user_token, created_at),
ALGORITHM=INPLACE, LOCK=NONE;
@@ -13,9 +13,18 @@ CREATE TABLE debug_logs (
-- The type of debug log entry.
debug_log_type VARCHAR(24) NOT NULL,
-- The severity level of the log entry (info, warn, error, debug, trace).
maybe_log_level VARCHAR(16) DEFAULT NULL,
-- The user who created the log entry (if any).
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
-- The client IP address of the request (if any). Wide enough for IPv6.
maybe_ip_address VARCHAR(40) DEFAULT NULL,
-- The request URL (if any). Truncated to 255 characters on insert.
maybe_url VARCHAR(255) DEFAULT NULL,
-- The log message body.
message MEDIUMTEXT NOT NULL,
@@ -25,6 +34,7 @@ CREATE TABLE debug_logs (
PRIMARY KEY (id),
KEY index_event_token (event_token),
KEY index_debug_log_type (debug_log_type),
KEY index_maybe_log_level (maybe_log_level),
KEY index_maybe_creator_user_token (maybe_creator_user_token),
KEY index_created_at (created_at)
@@ -364,6 +364,7 @@ CREATE TABLE media_files (
KEY index_user_deleted_at (user_deleted_at),
KEY index_mod_deleted_at (mod_deleted_at),
KEY index_media_files_created_at (created_at),
KEY index_media_files_updated_at (updated_at)
KEY index_media_files_updated_at (updated_at),
KEY idx_creator_created_at (maybe_creator_user_token, created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,36 @@
use chrono::{DateTime, Utc};
use serde_derive::Serialize;
use utoipa::ToSchema;
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
/// A single debug log row.
/// Shared by the moderation debug-log list endpoints.
#[derive(Serialize, ToSchema)]
pub struct ModerationDebugLogEntry {
pub id: u64,
pub event_token: DebugLogEventToken,
pub debug_log_type: DebugLogType,
pub maybe_log_level: Option<DebugLogLevel>,
pub maybe_creator_user_token: Option<UserToken>,
/// The client IP address of the request (if recorded).
pub maybe_ip_address: Option<String>,
/// The request URL (if recorded; truncated to 255 chars).
pub maybe_url: Option<String>,
pub message: String,
pub created_at: DateTime<Utc>,
/// The creator user, joined in when the row has a user token.
pub maybe_user: Option<ModerationDebugLogUser>,
}
/// The creator user of a debug log row.
#[derive(Serialize, ToSchema)]
pub struct ModerationDebugLogUser {
pub user_token: UserToken,
pub display_name: String,
pub username: String,
pub gravatar_hash: String,
}
@@ -0,0 +1,4 @@
pub mod debug_log_entry;
pub mod moderation_list_all_debug_logs;
pub mod moderation_list_debug_logs_for_token;
pub mod moderation_list_debug_logs_for_user;
@@ -0,0 +1,76 @@
use chrono::{DateTime, Utc};
use serde_derive::{Deserialize, Serialize};
use utoipa::{IntoParams, ToSchema};
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use enums::error::enum_error::EnumError;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
use crate::moderation::debug_logs::debug_log_entry::ModerationDebugLogUser;
pub const MODERATION_LIST_ALL_DEBUG_LOGS_PATH: &str = "/v1/moderation/debug_logs/list_all";
// ── Query params ──
#[derive(Deserialize, IntoParams, ToSchema)]
pub struct ModerationListAllDebugLogsQueryParams {
/// Optional comma-separated list of severity levels to include,
/// eg. `severities=error,warn`. Omit to include all rows.
pub severities: Option<String>,
/// Cursor for pagination. Pass the `next_cursor` from a previous response.
pub cursor: Option<u64>,
/// Max rows per page (default 50, max 200).
pub limit: Option<u32>,
}
impl ModerationListAllDebugLogsQueryParams {
/// Parse the comma-separated `severities` parameter.
/// Returns `None` when the parameter is absent or empty.
pub fn parsed_severities(&self) -> Result<Option<Vec<DebugLogLevel>>, EnumError> {
let raw = match self.severities.as_deref() {
Some(value) if !value.trim().is_empty() => value,
_ => return Ok(None),
};
let levels = raw
.split(',')
.map(str::trim)
.filter(|part| !part.is_empty())
.map(DebugLogLevel::from_str)
.collect::<Result<Vec<DebugLogLevel>, EnumError>>()?;
Ok(if levels.is_empty() { None } else { Some(levels) })
}
}
// ── Response ──
#[derive(Serialize, ToSchema)]
pub struct ModerationListAllDebugLogsSuccessResponse {
pub success: bool,
/// Most recent first.
pub debug_logs: Vec<ModerationListAllDebugLogsEntry>,
/// Cursor for the next page. `None` if there are no more results.
pub next_cursor: Option<u64>,
}
/// A debug log row with the creator user joined in (when one exists).
#[derive(Serialize, ToSchema)]
pub struct ModerationListAllDebugLogsEntry {
pub id: u64,
pub event_token: DebugLogEventToken,
pub debug_log_type: DebugLogType,
pub maybe_log_level: Option<DebugLogLevel>,
pub maybe_creator_user_token: Option<UserToken>,
/// The client IP address of the request (if recorded).
pub maybe_ip_address: Option<String>,
/// The request URL (if recorded; truncated to 255 chars).
pub maybe_url: Option<String>,
pub message: String,
pub created_at: DateTime<Utc>,
pub maybe_user: Option<ModerationDebugLogUser>,
}
@@ -0,0 +1,30 @@
use serde_derive::{Deserialize, Serialize};
use utoipa::{IntoParams, ToSchema};
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use crate::moderation::debug_logs::debug_log_entry::ModerationDebugLogEntry;
pub const MODERATION_LIST_DEBUG_LOGS_FOR_TOKEN_PATH: &str = "/v1/moderation/debug_logs/list/{token}";
// ── Path params ──
#[derive(Deserialize, ToSchema)]
pub struct ModerationListDebugLogsForTokenPathInfo {
pub token: DebugLogEventToken,
}
// ── Query params ──
#[derive(Deserialize, IntoParams, ToSchema)]
pub struct ModerationListDebugLogsForTokenQueryParams {
pub limit: Option<u32>,
}
// ── Response ──
#[derive(Serialize, ToSchema)]
pub struct ModerationListDebugLogsForTokenSuccessResponse {
pub success: bool,
pub debug_logs: Vec<ModerationDebugLogEntry>,
}
@@ -0,0 +1,37 @@
use serde_derive::{Deserialize, Serialize};
use utoipa::{IntoParams, ToSchema};
use tokens::tokens::users::UserToken;
use crate::moderation::debug_logs::debug_log_entry::ModerationDebugLogEntry;
pub const MODERATION_LIST_DEBUG_LOGS_FOR_USER_PATH: &str = "/v1/moderation/debug_logs/user_list/{user_token}";
// ── Path params ──
#[derive(Deserialize, ToSchema)]
pub struct ModerationListDebugLogsForUserPathInfo {
pub user_token: UserToken,
}
// ── Query params ──
#[derive(Deserialize, IntoParams, ToSchema)]
pub struct ModerationListDebugLogsForUserQueryParams {
/// Cursor for pagination. Pass the `next_cursor` from a previous response.
pub cursor: Option<u64>,
/// Max rows per page (default 50, max 200).
pub limit: Option<u32>,
}
// ── Response ──
#[derive(Serialize, ToSchema)]
pub struct ModerationListDebugLogsForUserSuccessResponse {
pub success: bool,
/// Most recent first.
pub debug_logs: Vec<ModerationDebugLogEntry>,
/// Cursor for the next page. `None` if there are no more results.
pub next_cursor: Option<u64>,
}
@@ -1,4 +1,6 @@
pub mod alerts;
pub mod debug_logs;
pub mod top_spenders;
pub mod jobs;
pub mod user;
pub mod user_referrals;
@@ -0,0 +1,82 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::users::UserToken;
use utoipa::{IntoParams, ToSchema};
pub const MODERATOR_LIST_TOP_SPENDERS_PATH: &str =
"/v1/moderation/top_spenders/list";
/// Rolling aggregation window for the top spenders list.
#[derive(Deserialize, Serialize, Clone, Copy, ToSchema)]
pub enum TopSpendersWindow {
#[serde(rename = "24h")]
TwentyFourHours,
#[serde(rename = "7d")]
SevenDays,
#[serde(rename = "14d")]
FourteenDays,
#[serde(rename = "30d")]
ThirtyDays,
#[serde(rename = "60d")]
SixtyDays,
#[serde(rename = "90d")]
NinetyDays,
#[serde(rename = "180d")]
OneHundredEightyDays,
#[serde(rename = "365d")]
ThreeHundredSixtyFiveDays,
}
impl TopSpendersWindow {
/// Window length in hours.
pub fn as_hours(self) -> i64 {
match self {
TopSpendersWindow::TwentyFourHours => 24,
TopSpendersWindow::SevenDays => 7 * 24,
TopSpendersWindow::FourteenDays => 14 * 24,
TopSpendersWindow::ThirtyDays => 30 * 24,
TopSpendersWindow::SixtyDays => 60 * 24,
TopSpendersWindow::NinetyDays => 90 * 24,
TopSpendersWindow::OneHundredEightyDays => 180 * 24,
TopSpendersWindow::ThreeHundredSixtyFiveDays => 365 * 24,
}
}
}
#[derive(Deserialize, IntoParams)]
pub struct ModeratorListTopSpendersQueryParams {
/// Aggregation window: `24h`, `7d`, `14d`, `30d` (default), `60d`, `90d`,
/// `180d`, or `365d`.
pub window: Option<TopSpendersWindow>,
/// Row offset for pagination (page size is fixed at 100). Pass `next_offset` from the previous page.
pub offset: Option<u64>,
/// Optional namespace filter (e.g. `artcraft`). Omit to return all namespaces.
pub payments_namespace: Option<String>,
}
#[derive(Serialize, ToSchema)]
pub struct ModeratorListTopSpendersResponse {
pub success: bool,
/// The window the results were aggregated over (echoed back, post-default).
pub window: TopSpendersWindow,
pub spenders: Vec<TopSpenderEntry>,
/// Offset for the next page, or `None` if this was the last page.
pub maybe_next_offset: Option<u64>,
}
#[derive(Serialize, ToSchema)]
pub struct TopSpenderEntry {
pub user_token: UserToken,
pub username: String,
pub display_name: String,
pub email_gravatar_hash: String,
/// Sum of positive amounts in the window.
pub gross_spend_usd_cents: u64,
/// Refund/chargeback magnitude in the window (positive number).
pub refund_usd_cents: u64,
/// Gross minus refunds. Can be negative if refunds outweigh purchases.
pub net_spend_usd_cents: i64,
/// Number of positive money events in the window.
pub payment_count: u64,
/// Credits granted in the window.
pub credits_granted: u64,
}
@@ -0,0 +1 @@
pub mod list;
@@ -82,6 +82,7 @@ fn do_build_cors_config(is_production: bool) -> Cors {
actix_http::header::ACCESS_CONTROL_ALLOW_CREDENTIALS, // https://stackoverflow.com/a/46412839
actix_http::header::HeaderName::from_static("x-requested-with"), // Tabulator Ajax sends
actix_http::header::HeaderName::from_static("session"), // Custom header sent by Three.js Storyteller Studio
actix_http::header::HeaderName::from_static("x-artcraft-version"), // ArtCraft client app release, surfaced in error alerts
])
.max_age(3600)
}
+1
View File
@@ -25,6 +25,7 @@ workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-
# External
dotenv.workspace = true
env_logger = "0.11.3"
trace_id.workspace = true
log = "0.4.14"
[dev-dependencies]
+32 -1
View File
@@ -29,6 +29,37 @@ pub fn init_env_logger(default_if_absent: Option<&str>) {
env::set_var(ENV_RUST_LOG, default_log_level);
}
env_logger::init();
// Custom format so that log lines emitted while serving an HTTP request
// automatically carry the request's trace id (see the `trace_id` crate and
// the `TraceIdMiddleware` in storyteller-web). Outside of a request scope
// the trace segment is simply omitted.
env_logger::Builder::from_env(env_logger::Env::default())
.format(|buf, record| {
use std::io::Write;
let timestamp = buf.timestamp();
let level_style = buf.default_level_style(record.level());
match trace_id::current_trace_id() {
Some(trace) => writeln!(
buf,
"[{} {level_style}{}{level_style:#} {} {}] {}",
timestamp,
record.level(),
record.target(),
trace,
record.args(),
),
None => writeln!(
buf,
"[{} {level_style}{}{level_style:#} {}] {}",
timestamp,
record.level(),
record.target(),
record.args(),
),
}
})
.init();
}
+1
View File
@@ -25,6 +25,7 @@ workspace-build-acceleration = { workspace = true }
# External
env_logger = "0.11.3"
trace_id.workspace = true
log = { workspace = true }
[dev-dependencies]
+32 -1
View File
@@ -68,6 +68,37 @@ pub fn init_env_logger(default_if_absent: Option<&str>) {
env::set_var(ENV_RUST_LOG, default_log_level);
}
env_logger::init();
// Custom format so that log lines emitted while serving an HTTP request
// automatically carry the request's trace id (see the `trace_id` crate and
// the `TraceIdMiddleware` in storyteller-web). Outside of a request scope
// the trace segment is simply omitted.
env_logger::Builder::from_env(env_logger::Env::default())
.format(|buf, record| {
use std::io::Write;
let timestamp = buf.timestamp();
let level_style = buf.default_level_style(record.level());
match trace_id::current_trace_id() {
Some(trace) => writeln!(
buf,
"[{} {level_style}{}{level_style:#} {} {}] {}",
timestamp,
record.level(),
record.target(),
trace,
record.args(),
),
None => writeln!(
buf,
"[{} {level_style}{}{level_style:#} {}] {}",
timestamp,
record.level(),
record.target(),
record.args(),
),
}
})
.init();
}
+1
View File
@@ -24,6 +24,7 @@ chrono.workspace = true
sha2.workspace = true
log.workspace = true
tokio.workspace = true
trace_id.workspace = true
[dev-dependencies]
tokio = { version = "1", features = ["full"] }
@@ -63,6 +63,12 @@ mod tests {
media_file_token: None,
inference_job_token: None,
third_party_id: None,
http_host: None,
http_origin: None,
http_referer: None,
http_user_agent: None,
artcraft_version: None,
trace_id: None,
request_ip_address: None,
avt_cookie_token: None,
session_token: None,
@@ -228,6 +234,12 @@ mod tests {
media_file_token: None,
inference_job_token: None,
third_party_id: None,
http_host: None,
http_origin: None,
http_referer: None,
http_user_agent: None,
artcraft_version: None,
trace_id: None,
request_ip_address: None,
avt_cookie_token: None,
session_token: None,
@@ -37,6 +37,27 @@ pub struct NotificationDetails {
/// HTTP status code associated with the event, if any.
pub(crate) http_status_code: Option<u16>,
/// Hostname the client used to reach this server (from the `Host` or
/// forwarded-host headers), if any. Useful when a server is reachable
/// via multiple domains (eg. api.storyteller.ai, api.fakeyou.com).
pub(crate) http_host: Option<String>,
/// `Origin` header of the request, if any. For browser XHR/fetch this is
/// the scheme+host of the calling frontend.
pub(crate) http_origin: Option<String>,
/// `Referer` header of the request, if any. For browser XHR/fetch this is
/// the URL of the page that made the call (possibly truncated to the
/// origin by the browser's referrer policy).
pub(crate) http_referer: Option<String>,
/// `User-Agent` header of the request, if any.
pub(crate) http_user_agent: Option<String>,
/// `X-ArtCraft-Version` header of the request, if any. Sent by ArtCraft
/// clients to identify which app release made the call.
pub(crate) artcraft_version: Option<String>,
/// User token associated with the event, if any.
pub(crate) user_token: Option<String>,
@@ -49,6 +70,9 @@ pub struct NotificationDetails {
/// Third-party identifier associated with the event, if any.
pub(crate) third_party_id: Option<String>,
/// The request-scoped trace id associated with the event, if any.
pub(crate) trace_id: Option<String>,
/// IP address of the request that triggered this notification, if any.
pub(crate) request_ip_address: Option<String>,
@@ -185,10 +209,34 @@ impl NotificationDetails {
parts.push(format!("HTTP Status Code: {}", status_code));
}
if let Some(host) = &self.http_host {
parts.push(format!("HTTP Host: {}", host));
}
if let Some(origin) = &self.http_origin {
parts.push(format!("HTTP Origin: {}", origin));
}
if let Some(referer) = &self.http_referer {
parts.push(format!("HTTP Referer: {}", referer));
}
if let Some(user_agent) = &self.http_user_agent {
parts.push(format!("User Agent: {}", user_agent));
}
if let Some(version) = &self.artcraft_version {
parts.push(format!("ArtCraft Version: {}", version));
}
if let Some(ip) = &self.request_ip_address {
parts.push(format!("Request IP: {}", ip));
}
if let Some(trace_id) = &self.trace_id {
parts.push(format!("Trace ID: {}", trace_id));
}
if parts.is_empty() { None } else { Some(parts.join("\n")) }
}
@@ -21,6 +21,11 @@ pub struct NotificationDetailsBuilder {
pub(crate) http_method: Option<String>,
pub(crate) http_path: Option<String>,
pub(crate) http_status_code: Option<u16>,
pub(crate) http_host: Option<String>,
pub(crate) http_origin: Option<String>,
pub(crate) http_referer: Option<String>,
pub(crate) http_user_agent: Option<String>,
pub(crate) artcraft_version: Option<String>,
pub(crate) user_token: Option<String>,
pub(crate) media_file_token: Option<String>,
@@ -31,6 +36,10 @@ pub struct NotificationDetailsBuilder {
pub(crate) avt_cookie_token: Option<String>,
pub(crate) session_token: Option<String>,
pub(crate) session_user_token: Option<String>,
/// The request-scoped trace id, if any. Auto-filled from the tokio
/// task-local at `build()` time when not explicitly set.
pub(crate) trace_id: Option<String>,
}
impl NotificationDetailsBuilder {
@@ -49,6 +58,11 @@ impl NotificationDetailsBuilder {
http_method: None,
http_path: None,
http_status_code: None,
http_host: None,
http_origin: None,
http_referer: None,
http_user_agent: None,
artcraft_version: None,
user_token: None,
media_file_token: None,
inference_job_token: None,
@@ -57,6 +71,7 @@ impl NotificationDetailsBuilder {
avt_cookie_token: None,
session_token: None,
session_user_token: None,
trace_id: None,
}
}
@@ -79,6 +94,11 @@ impl NotificationDetailsBuilder {
http_method: None,
http_path: None,
http_status_code: None,
http_host: None,
http_origin: None,
http_referer: None,
http_user_agent: None,
artcraft_version: None,
user_token: None,
media_file_token: None,
inference_job_token: None,
@@ -87,6 +107,7 @@ impl NotificationDetailsBuilder {
avt_cookie_token: None,
session_token: None,
session_user_token: None,
trace_id: None,
}
}
@@ -132,6 +153,31 @@ impl NotificationDetailsBuilder {
self
}
pub fn set_http_host(mut self, http_host: Option<String>) -> Self {
self.http_host = http_host;
self
}
pub fn set_http_origin(mut self, http_origin: Option<String>) -> Self {
self.http_origin = http_origin;
self
}
pub fn set_http_referer(mut self, http_referer: Option<String>) -> Self {
self.http_referer = http_referer;
self
}
pub fn set_http_user_agent(mut self, http_user_agent: Option<String>) -> Self {
self.http_user_agent = http_user_agent;
self
}
pub fn set_artcraft_version(mut self, artcraft_version: Option<String>) -> Self {
self.artcraft_version = artcraft_version;
self
}
pub fn set_user_token(mut self, user_token: Option<String>) -> Self {
self.user_token = user_token;
self
@@ -167,6 +213,11 @@ impl NotificationDetailsBuilder {
self
}
pub fn set_trace_id(mut self, trace_id: Option<String>) -> Self {
self.trace_id = trace_id;
self
}
pub fn set_session_user_token(mut self, session_user_token: Option<String>) -> Self {
self.session_user_token = session_user_token;
self
@@ -191,6 +242,11 @@ impl NotificationDetailsBuilder {
http_method: self.http_method,
http_path: self.http_path,
http_status_code: self.http_status_code,
http_host: self.http_host,
http_origin: self.http_origin,
http_referer: self.http_referer,
http_user_agent: self.http_user_agent,
artcraft_version: self.artcraft_version,
user_token: self.user_token,
media_file_token: self.media_file_token,
inference_job_token: self.inference_job_token,
@@ -199,6 +255,11 @@ impl NotificationDetailsBuilder {
avt_cookie_token: self.avt_cookie_token,
session_token: self.session_token,
session_user_token: self.session_user_token,
// Auto-fill from the request's task-local so any page fired while
// serving a request is correlated even if the caller never set it.
trace_id: self.trace_id.or_else(|| {
trace_id::current_trace_id().map(|t| t.to_string())
}),
}
}
@@ -3,6 +3,7 @@ use std::marker::PhantomData;
use log::warn;
use sqlx::{Executor, MySql, QueryBuilder};
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
@@ -18,6 +19,12 @@ where
/// The user associated with these debug log events (if any).
pub maybe_creator_user_token: Option<&'e UserToken>,
/// The client IP address of the request (if any). Truncated on insert.
pub maybe_ip_address: Option<&'e str>,
/// The request URL (if any). Truncated to 255 characters on insert.
pub maybe_url: Option<&'e str>,
/// The events to insert.
pub events: Vec<DebugLogEvent>,
@@ -31,6 +38,7 @@ where
/// Callers should use `serde_json::to_string(...)` before constructing this.
pub struct DebugLogEvent {
pub debug_log_type: DebugLogType,
pub maybe_log_level: Option<DebugLogLevel>,
pub message: String,
}
@@ -54,9 +62,11 @@ where
}
let maybe_user_token_str = args.maybe_creator_user_token.map(|t| t.as_str().to_string());
let maybe_ip_address = args.maybe_ip_address.map(|ip| truncate_chars(ip, MAX_IP_ADDRESS_CHARS));
let maybe_url = args.maybe_url.map(|url| truncate_chars(url, MAX_URL_CHARS));
let mut query_builder = QueryBuilder::new(
"INSERT INTO debug_logs (event_token, debug_log_type, maybe_creator_user_token, message) VALUES "
"INSERT INTO debug_logs (event_token, debug_log_type, maybe_log_level, maybe_creator_user_token, maybe_ip_address, maybe_url, message) VALUES "
);
for (i, event) in args.events.iter().enumerate() {
@@ -65,8 +75,14 @@ where
query_builder.push(", ");
query_builder.push_bind(event.debug_log_type.to_str());
query_builder.push(", ");
query_builder.push_bind(event.maybe_log_level.map(|l| l.to_str()));
query_builder.push(", ");
query_builder.push_bind(&maybe_user_token_str);
query_builder.push(", ");
query_builder.push_bind(maybe_ip_address);
query_builder.push(", ");
query_builder.push_bind(maybe_url);
query_builder.push(", ");
query_builder.push_bind(&event.message);
query_builder.push(")");
@@ -81,3 +97,17 @@ where
Ok(event_token)
}
/// Maximum stored length for `maybe_ip_address` (VARCHAR(40)).
const MAX_IP_ADDRESS_CHARS: usize = 40;
/// Maximum stored length for `maybe_url` (VARCHAR(255)).
const MAX_URL_CHARS: usize = 255;
/// Truncate to at most `max_chars` characters (UTF-8 safe).
fn truncate_chars(value: &str, max_chars: usize) -> &str {
match value.char_indices().nth(max_chars) {
Some((idx, _)) => &value[..idx],
None => value,
}
}
@@ -2,6 +2,7 @@ use std::marker::PhantomData;
use sqlx::{Executor, MySql};
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
@@ -20,9 +21,18 @@ where
/// The type of debug log event.
pub debug_log_type: DebugLogType,
/// The severity level of the log event (if any).
pub maybe_log_level: Option<DebugLogLevel>,
/// The log message body (pre-serialized JSON or any text).
pub message: &'e str,
/// The client IP address of the request (if any). Truncated on insert.
pub maybe_ip_address: Option<&'e str>,
/// The request URL (if any). Truncated to 255 characters on insert.
pub maybe_url: Option<&'e str>,
pub mysql_executor: E,
pub phantom: PhantomData<&'c E>,
}
@@ -47,12 +57,18 @@ INSERT INTO debug_logs
SET
event_token = ?,
debug_log_type = ?,
maybe_log_level = ?,
maybe_creator_user_token = ?,
maybe_ip_address = ?,
maybe_url = ?,
message = ?
"#,
event_token.as_str(),
args.debug_log_type.to_str(),
args.maybe_log_level.map(|l| l.to_str()),
args.maybe_creator_user_token.map(|t| t.as_str()),
args.maybe_ip_address.map(|ip| truncate_chars(ip, MAX_IP_ADDRESS_CHARS)),
args.maybe_url.map(|url| truncate_chars(url, MAX_URL_CHARS)),
args.message,
)
.execute(args.mysql_executor)
@@ -60,3 +76,17 @@ SET
Ok(event_token)
}
/// Maximum stored length for `maybe_ip_address` (VARCHAR(40)).
const MAX_IP_ADDRESS_CHARS: usize = 40;
/// Maximum stored length for `maybe_url` (VARCHAR(255)).
const MAX_URL_CHARS: usize = 255;
/// Truncate to at most `max_chars` characters (UTF-8 safe).
fn truncate_chars(value: &str, max_chars: usize) -> &str {
match value.char_indices().nth(max_chars) {
Some((idx, _)) => &value[..idx],
None => value,
}
}
@@ -0,0 +1,148 @@
use std::marker::PhantomData;
use chrono::{DateTime, Utc};
use sqlx::mysql::MySqlRow;
use sqlx::{Executor, MySql, QueryBuilder, Row};
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
const DEFAULT_LIMIT: u32 = 50;
const MAX_LIMIT: u32 = 200;
pub struct ListAllDebugLogsArgs<'c, E>
where
E: Executor<'c, Database = MySql>,
{
/// If provided, only rows with one of these log levels are returned.
/// (Rows with a NULL log level are excluded when filtering.)
pub maybe_log_levels: Option<Vec<DebugLogLevel>>,
/// Cursor for pagination: only rows with `id` strictly below this are
/// returned. Pass the `next_cursor` from a previous page.
pub maybe_id_cursor: Option<u64>,
pub limit: Option<u32>,
pub mysql_executor: E,
pub phantom: PhantomData<&'c E>,
}
pub struct ListAllDebugLogsResult {
pub debug_logs: Vec<AllDebugLogRow>,
/// Cursor for the next page. `None` if there are no more results.
pub next_cursor: Option<u64>,
}
/// A debug log row left-outer-joined against the creator user (if any).
#[derive(Debug)]
pub struct AllDebugLogRow {
pub id: u64,
pub event_token: DebugLogEventToken,
pub debug_log_type: DebugLogType,
pub maybe_log_level: Option<DebugLogLevel>,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_ip_address: Option<String>,
pub maybe_url: Option<String>,
pub message: String,
pub created_at: DateTime<Utc>,
// Joined user fields (present when the creator user exists).
pub maybe_user_display_name: Option<String>,
pub maybe_user_username: Option<String>,
pub maybe_user_gravatar_hash: Option<String>,
}
/// List all debug logs, most recent first, cursor-paginated by `id`,
/// optionally scoped to a set of log levels.
///
/// NB: Uses `QueryBuilder` (not the compile-time macros) because of the
/// dynamic `WHERE maybe_log_level IN (...)` clause.
pub async fn list_all_debug_logs<'c, E>(
args: ListAllDebugLogsArgs<'c, E>,
) -> Result<ListAllDebugLogsResult, sqlx::Error>
where
E: Executor<'c, Database = MySql>,
{
let limit = args.limit.unwrap_or(DEFAULT_LIMIT).min(MAX_LIMIT);
// Fetch limit + 1 so we can detect whether there's a next page.
let fetch_limit = (limit + 1) as i64;
let id_cursor = args.maybe_id_cursor.unwrap_or(u64::MAX);
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
r#"
SELECT
d.id,
d.event_token,
d.debug_log_type,
d.maybe_log_level,
d.maybe_creator_user_token,
d.maybe_ip_address,
d.maybe_url,
d.message,
d.created_at,
u.display_name as maybe_user_display_name,
u.username as maybe_user_username,
u.email_gravatar_hash as maybe_user_gravatar_hash
FROM debug_logs d
LEFT OUTER JOIN users u
ON u.token = d.maybe_creator_user_token
WHERE d.id < "#,
);
query_builder.push_bind(id_cursor);
if let Some(levels) = args.maybe_log_levels.as_ref().filter(|l| !l.is_empty()) {
query_builder.push(" AND d.maybe_log_level IN (");
let mut separated = query_builder.separated(", ");
for level in levels {
separated.push_bind(level.to_str());
}
separated.push_unseparated(")");
}
query_builder.push(" ORDER BY d.id DESC LIMIT ");
query_builder.push_bind(fetch_limit);
let raw_rows: Vec<MySqlRow> = query_builder
.build()
.fetch_all(args.mysql_executor)
.await?;
let mut debug_logs = raw_rows
.into_iter()
.map(row_from_mysql_row)
.collect::<Result<Vec<AllDebugLogRow>, sqlx::Error>>()?;
let next_cursor = if debug_logs.len() > limit as usize {
debug_logs.truncate(limit as usize);
debug_logs.last().map(|row| row.id)
} else {
None
};
Ok(ListAllDebugLogsResult {
debug_logs,
next_cursor,
})
}
fn row_from_mysql_row(row: MySqlRow) -> Result<AllDebugLogRow, sqlx::Error> {
let created_at: DateTime<Utc> = row.try_get("created_at")?;
Ok(AllDebugLogRow {
id: row.try_get("id")?,
event_token: row.try_get("event_token")?,
debug_log_type: row.try_get("debug_log_type")?,
maybe_log_level: row.try_get("maybe_log_level")?,
maybe_creator_user_token: row.try_get("maybe_creator_user_token")?,
maybe_ip_address: row.try_get("maybe_ip_address")?,
maybe_url: row.try_get("maybe_url")?,
message: row.try_get("message")?,
created_at,
maybe_user_display_name: row.try_get("maybe_user_display_name")?,
maybe_user_username: row.try_get("maybe_user_username")?,
maybe_user_gravatar_hash: row.try_get("maybe_user_gravatar_hash")?,
})
}
@@ -3,6 +3,7 @@ use std::marker::PhantomData;
use chrono::{DateTime, NaiveDateTime, Utc};
use sqlx::{Executor, MySql};
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
@@ -24,9 +25,17 @@ pub struct DebugLogRow {
pub id: u64,
pub event_token: DebugLogEventToken,
pub debug_log_type: DebugLogType,
pub maybe_log_level: Option<DebugLogLevel>,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_ip_address: Option<String>,
pub maybe_url: Option<String>,
pub message: String,
pub created_at: DateTime<Utc>,
// Joined user fields (present when the creator user exists).
pub maybe_user_display_name: Option<String>,
pub maybe_user_username: Option<String>,
pub maybe_user_gravatar_hash: Option<String>,
}
#[derive(Debug)]
@@ -34,9 +43,15 @@ struct RawDebugLogRow {
id: u64,
event_token: DebugLogEventToken,
debug_log_type: DebugLogType,
maybe_log_level: Option<DebugLogLevel>,
maybe_creator_user_token: Option<UserToken>,
maybe_ip_address: Option<String>,
maybe_url: Option<String>,
message: String,
created_at: NaiveDateTime,
maybe_user_display_name: Option<String>,
maybe_user_username: Option<String>,
maybe_user_gravatar_hash: Option<String>,
}
pub async fn list_debug_logs_for_token<'e, 'c: 'e, E>(
@@ -51,15 +66,23 @@ where
RawDebugLogRow,
r#"
SELECT
id as `id: u64`,
event_token as `event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,
debug_log_type as `debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType`,
maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
message,
created_at
FROM debug_logs
WHERE event_token = ?
ORDER BY id ASC
d.id as `id: u64`,
d.event_token as `event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,
d.debug_log_type as `debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType`,
d.maybe_log_level as `maybe_log_level: enums::by_table::debug_logs::debug_log_level::DebugLogLevel`,
d.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
d.maybe_ip_address,
d.maybe_url,
d.message,
d.created_at,
u.display_name as `maybe_user_display_name?`,
u.username as `maybe_user_username?`,
u.email_gravatar_hash as `maybe_user_gravatar_hash?`
FROM debug_logs d
LEFT OUTER JOIN users u
ON u.token = d.maybe_creator_user_token
WHERE d.event_token = ?
ORDER BY d.id ASC
LIMIT ?
"#,
args.event_token.as_str(),
@@ -73,9 +96,15 @@ LIMIT ?
id: row.id,
event_token: row.event_token,
debug_log_type: row.debug_log_type,
maybe_log_level: row.maybe_log_level,
maybe_creator_user_token: row.maybe_creator_user_token,
maybe_ip_address: row.maybe_ip_address,
maybe_url: row.maybe_url,
message: row.message,
created_at: row.created_at.and_utc(),
maybe_user_display_name: row.maybe_user_display_name,
maybe_user_username: row.maybe_user_username,
maybe_user_gravatar_hash: row.maybe_user_gravatar_hash,
}
}).collect();
@@ -0,0 +1,142 @@
use std::marker::PhantomData;
use chrono::{DateTime, NaiveDateTime, Utc};
use sqlx::{Executor, MySql};
use enums::by_table::debug_logs::debug_log_level::DebugLogLevel;
use enums::by_table::debug_logs::debug_log_type::DebugLogType;
use tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken;
use tokens::tokens::users::UserToken;
const DEFAULT_LIMIT: u32 = 50;
const MAX_LIMIT: u32 = 200;
pub struct ListDebugLogsForUserArgs<'e, 'c, E>
where
E: 'e + Executor<'c, Database = MySql>,
{
pub user_token: &'e UserToken,
/// Cursor for pagination: only rows with `id` strictly below this are
/// returned. Pass the `next_cursor` from a previous page.
pub maybe_id_cursor: Option<u64>,
pub limit: Option<u32>,
pub mysql_executor: E,
pub phantom: PhantomData<&'c E>,
}
pub struct ListDebugLogsForUserResult {
pub debug_logs: Vec<UserDebugLogRow>,
/// Cursor for the next page. `None` if there are no more results.
pub next_cursor: Option<u64>,
}
#[derive(Debug)]
pub struct UserDebugLogRow {
pub id: u64,
pub event_token: DebugLogEventToken,
pub debug_log_type: DebugLogType,
pub maybe_log_level: Option<DebugLogLevel>,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_ip_address: Option<String>,
pub maybe_url: Option<String>,
pub message: String,
pub created_at: DateTime<Utc>,
// Joined user fields (present when the creator user exists).
pub maybe_user_display_name: Option<String>,
pub maybe_user_username: Option<String>,
pub maybe_user_gravatar_hash: Option<String>,
}
#[derive(Debug)]
struct RawUserDebugLogRow {
id: u64,
event_token: DebugLogEventToken,
debug_log_type: DebugLogType,
maybe_log_level: Option<DebugLogLevel>,
maybe_creator_user_token: Option<UserToken>,
maybe_ip_address: Option<String>,
maybe_url: Option<String>,
message: String,
created_at: NaiveDateTime,
maybe_user_display_name: Option<String>,
maybe_user_username: Option<String>,
maybe_user_gravatar_hash: Option<String>,
}
/// List debug logs for a user, most recent first, cursor-paginated by `id`.
pub async fn list_debug_logs_for_user<'e, 'c: 'e, E>(
args: ListDebugLogsForUserArgs<'e, 'c, E>,
) -> Result<ListDebugLogsForUserResult, sqlx::Error>
where
E: 'e + Executor<'c, Database = MySql>,
{
let limit = args.limit.unwrap_or(DEFAULT_LIMIT).min(MAX_LIMIT);
// Fetch limit + 1 so we can detect whether there's a next page.
let fetch_limit = (limit + 1) as i64;
// Use u64::MAX when no cursor so all rows are included.
let id_cursor = args.maybe_id_cursor.unwrap_or(u64::MAX);
let rows = sqlx::query_as!(
RawUserDebugLogRow,
r#"
SELECT
d.id as `id: u64`,
d.event_token as `event_token: tokens::tokens::non_unique::debug_logs_event_token::DebugLogEventToken`,
d.debug_log_type as `debug_log_type: enums::by_table::debug_logs::debug_log_type::DebugLogType`,
d.maybe_log_level as `maybe_log_level: enums::by_table::debug_logs::debug_log_level::DebugLogLevel`,
d.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
d.maybe_ip_address,
d.maybe_url,
d.message,
d.created_at,
u.display_name as `maybe_user_display_name?`,
u.username as `maybe_user_username?`,
u.email_gravatar_hash as `maybe_user_gravatar_hash?`
FROM debug_logs d
LEFT OUTER JOIN users u
ON u.token = d.maybe_creator_user_token
WHERE d.maybe_creator_user_token = ?
AND d.id < ?
ORDER BY d.id DESC
LIMIT ?
"#,
args.user_token.as_str(),
id_cursor,
fetch_limit,
)
.fetch_all(args.mysql_executor)
.await?;
let mut debug_logs: Vec<UserDebugLogRow> = rows.into_iter().map(|row| {
UserDebugLogRow {
id: row.id,
event_token: row.event_token,
debug_log_type: row.debug_log_type,
maybe_log_level: row.maybe_log_level,
maybe_creator_user_token: row.maybe_creator_user_token,
maybe_ip_address: row.maybe_ip_address,
maybe_url: row.maybe_url,
message: row.message,
created_at: row.created_at.and_utc(),
maybe_user_display_name: row.maybe_user_display_name,
maybe_user_username: row.maybe_user_username,
maybe_user_gravatar_hash: row.maybe_user_gravatar_hash,
}
}).collect();
let next_cursor = if debug_logs.len() > limit as usize {
debug_logs.truncate(limit as usize);
debug_logs.last().map(|row| row.id)
} else {
None
};
Ok(ListDebugLogsForUserResult {
debug_logs,
next_cursor,
})
}
@@ -1,3 +1,5 @@
pub mod bulk_insert_debug_logs;
pub mod insert_debug_log;
pub mod list_all_debug_logs;
pub mod list_debug_logs_for_token;
pub mod list_debug_logs_for_user;
@@ -107,6 +107,7 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An
args.sort_ascending,
args.view_as,
count_fields.as_str(),
true,
);
let row_count_query = count_query_builder.build_query_scalar::<i64>();
@@ -126,6 +127,7 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An
args.sort_ascending,
args.view_as,
result_fields.as_str(),
false,
);
let query = query.build_query_as::<MediaFileListItemInternal>();
@@ -250,13 +252,20 @@ fn query_builder<'a>(
sort_ascending: bool,
view_as: ViewAs,
select_fields: &'a str,
count_only: bool,
) -> QueryBuilder<'a, MySql> {
// NB: Query cannot be statically checked by sqlx
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
format!(r#"
SELECT
{select_fields}
// NB: The count query only needs the `users` join (for the username predicate).
// The other joins are all 1:1 and contribute nothing to the count — MySQL does
// not eliminate LEFT JOINs, so including them costs four index lookups per row.
let from_clause = if count_only {
r#"
FROM media_files AS m
LEFT OUTER JOIN users AS u
ON m.maybe_creator_user_token = u.token
"#
} else {
r#"
FROM media_files AS m
LEFT OUTER JOIN users AS u
ON m.maybe_creator_user_token = u.token
@@ -270,6 +279,14 @@ LEFT OUTER JOIN entity_stats
LEFT OUTER JOIN prompts
ON prompts.token = m.maybe_prompt_token
"#
};
// NB: Query cannot be statically checked by sqlx
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
format!(r#"
SELECT
{select_fields}
{from_clause}"#
));
query_builder.push(" WHERE u.username = ");
@@ -353,10 +370,13 @@ LEFT OUTER JOIN prompts
}
}
if sort_ascending {
query_builder.push(" ORDER BY m.created_at ASC ");
} else {
query_builder.push(" ORDER BY m.created_at DESC ");
// NB: Ordering is meaningless for the single-row COUNT aggregate.
if !count_only {
if sort_ascending {
query_builder.push(" ORDER BY m.created_at ASC ");
} else {
query_builder.push(" ORDER BY m.created_at DESC ");
}
}
if enforce_limits {
@@ -47,7 +47,6 @@ pub mod user_spend_events;
pub mod user_spend_summaries;
pub mod users;
pub mod vocoder;
pub mod voice_clone_requests;
pub mod voice_conversion;
pub mod voice_designer;
pub mod wallet_ledger_entries;
@@ -1,472 +0,0 @@
use std::collections::HashSet;
use chrono::{DateTime, Utc};
use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
use sqlx::mysql::MySqlRow;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::view_as::ViewAs;
use enums::traits::mysql_from_row::MySqlFromRow as _;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
pub struct FeaturedWeightsPage {
pub records: Vec<FeaturedWeight>,
pub sort_ascending: bool,
/// ID of the first record in the result set.
pub first_id: Option<i64>,
/// ID of the last record in the result set.
pub last_id: Option<i64>,
}
#[derive(Serialize)]
pub struct FeaturedWeight {
pub token: ModelWeightToken,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub title: String,
// TODO(bt,2023-12-24): These aren't really appropriate for a list endpoint.
// Hopefully we don't break the frontend by omitting these.
//pub maybe_description_markdown: String,
//pub maybe_description_rendered_html: String,
pub maybe_ietf_language_tag: Option<String>,
pub maybe_ietf_primary_language_subtag: Option<String>,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_creator_username: Option<String>,
pub maybe_creator_display_name: Option<String>,
pub maybe_creator_email_gravatar_hash: Option<String>,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub maybe_cover_image_public_bucket_hash: Option<String>,
pub maybe_cover_image_public_bucket_prefix: Option<String>,
pub maybe_cover_image_public_bucket_extension: Option<String>,
pub maybe_ratings_positive_count: Option<u32>,
pub maybe_ratings_negative_count: Option<u32>,
pub maybe_bookmark_count: Option<u32>,
pub cached_usage_count: u64,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
}
pub struct ListFeaturedWeightsArgs<'a> {
pub limit: usize,
pub maybe_offset: Option<usize>,
pub cursor_is_reversed: bool,
pub sort_ascending: bool,
pub view_as: ViewAs,
pub maybe_scoped_weight_types: Option<&'a HashSet<WeightsType>>,
pub maybe_scoped_weight_categories: Option<&'a HashSet<WeightsCategory>>,
pub mysql_pool: &'a MySqlPool,
}
pub async fn list_featured_weights(args: ListFeaturedWeightsArgs<'_>) -> AnyhowResult<FeaturedWeightsPage> {
// TODO(bt,2024-05-05): Why were we doing this, again?
// let count_fields = select_total_count_field();
// let mut count_query_builder = query_builder(
// args.limit,
// args.maybe_offset,
// args.cursor_is_reversed,
// args.sort_ascending,
// count_fields.as_str(),
// args.view_as,
// args.maybe_scoped_weight_type,
// args.maybe_scoped_weight_category,
// );
// let row_count_query = count_query_builder.build_query_scalar::<i64>();
// let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?;
/// Now fetch the actual results with all the fields
let result_fields = select_result_fields();
let mut query = query_builder(
args.limit,
args.maybe_offset,
args.cursor_is_reversed,
args.sort_ascending,
result_fields.as_str(),
args.view_as,
args.maybe_scoped_weight_types,
args.maybe_scoped_weight_categories,
);
let query = query.build_query_as::<RawWeightJoinUser>();
let results = query.fetch_all(args.mysql_pool).await?;
let first_id = results.first()
.map(|raw_result| raw_result.id);
let last_id = results.last()
.map(|raw_result| raw_result.id);
let weights_records: Vec<FeaturedWeight> = map_to_weights(results).await;
Ok(FeaturedWeightsPage {
records: weights_records,
sort_ascending: args.sort_ascending,
first_id,
last_id,
})
}
fn select_total_count_field() -> String {
r#"
COUNT(mw.id) AS total_count
"#
.to_string()
}
fn select_result_fields() -> String {
r#"
mw.id,
mw.token,
mw.title,
mw.weights_type,
mw.weights_category,
mw.maybe_ietf_language_tag,
mw.maybe_ietf_primary_language_subtag,
u.token as maybe_creator_user_token,
u.username as maybe_creator_username,
u.display_name as maybe_creator_display_name,
u.email_gravatar_hash as maybe_creator_email_gravatar_hash,
mw.creator_ip_address,
mw.creator_set_visibility,
mw.maybe_last_update_user_token,
mw.original_download_url,
mw.original_filename,
mw.file_size_bytes,
mw.file_checksum_sha2,
mw.public_bucket_hash,
mw.maybe_public_bucket_prefix,
mw.maybe_public_bucket_extension,
cover_image.public_bucket_directory_hash as maybe_cover_image_public_bucket_hash,
cover_image.maybe_public_bucket_prefix as maybe_cover_image_public_bucket_prefix,
cover_image.maybe_public_bucket_extension as maybe_cover_image_public_bucket_extension,
entity_stats.ratings_positive_count as maybe_ratings_positive_count,
entity_stats.ratings_negative_count as maybe_ratings_negative_count,
entity_stats.bookmark_count as maybe_bookmark_count,
mw.cached_usage_count,
mw.version,
mw.created_at,
mw.updated_at,
mw.user_deleted_at,
mw.mod_deleted_at
"#.to_string()
}
fn query_builder<'a>(
_limit: usize, // TODO(bt,2024-05-16): This seems wrong. Why is this unused?
maybe_offset: Option<usize>,
cursor_is_reversed: bool,
sort_ascending: bool,
select_fields: &'a str,
view_as: ViewAs,
maybe_scoped_weight_types: Option<&'a HashSet<WeightsType>>,
maybe_scoped_weight_categories: Option<&'a HashSet<WeightsCategory>>,
) -> QueryBuilder<'a, MySql> {
// NB: Query cannot be statically checked by sqlx
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
format!(r#"
SELECT
{select_fields}
FROM model_weights as mw
JOIN featured_items
ON featured_items.entity_type = "model_weight"
AND featured_items.deleted_at IS NULL
AND featured_items.entity_token = mw.token
LEFT OUTER JOIN users as u
ON u.token = mw.creator_user_token
LEFT OUTER JOIN media_files as cover_image
ON cover_image.token = mw.maybe_cover_image_media_file_token
LEFT OUTER JOIN entity_stats
ON entity_stats.entity_type = "model_weights"
AND entity_stats.entity_token = mw.token
"#
));
let mut first_predicate_added = false;
match view_as {
ViewAs::Author | ViewAs::Moderator => {
if !first_predicate_added {
query_builder.push(" WHERE ");
first_predicate_added = true;
} else {
query_builder.push(" AND ");
}
// NB(bt): Actually, mods don't want to see deleted models. We'll improve the moderator UI later.
query_builder.push(" mw.user_deleted_at IS NULL AND mw.mod_deleted_at IS NULL ");
}
ViewAs::AnotherUser => {
if !first_predicate_added {
query_builder.push(" WHERE ");
first_predicate_added = true;
} else {
query_builder.push(" AND ");
}
query_builder.push(" mw.user_deleted_at IS NULL AND mw.mod_deleted_at IS NULL ");
query_builder.push(" AND mw.creator_set_visibility = ");
query_builder.push_bind(Visibility::Public.to_str());
}
}
if let Some(weight_types) = maybe_scoped_weight_types {
// NB: `WHERE IN` comma separated syntax will be wrong if list has zero length
// We'll skip the predicate if the list isn't empty.
if !weight_types.is_empty() {
if !first_predicate_added {
query_builder.push(" WHERE ");
first_predicate_added = true;
} else {
query_builder.push(" AND ");
}
query_builder.push(" mw.weights_type IN ( ");
let mut separated = query_builder.separated(", ");
for weight_type in weight_types.iter() {
separated.push_bind(weight_type.to_str());
}
separated.push_unseparated(") ");
}
}
if let Some(weight_categories) = maybe_scoped_weight_categories {
// NB: `WHERE IN` comma separated syntax will be wrong if list has zero length
// We'll skip the predicate if the list isn't empty.
if !weight_categories.is_empty() {
if !first_predicate_added {
query_builder.push(" WHERE ");
first_predicate_added = true;
} else {
query_builder.push(" AND ");
}
query_builder.push(" mw.weights_category IN ( ");
let mut separated = query_builder.separated(", ");
for weights_category in weight_categories.iter() {
separated.push_bind(weights_category.to_str());
}
separated.push_unseparated(") ");
}
}
if let Some(offset) = maybe_offset {
if !first_predicate_added {
query_builder.push(" WHERE ");
first_predicate_added = true;
} else {
query_builder.push(" AND ");
}
let mut sort_ascending = sort_ascending;
if sort_ascending {
if cursor_is_reversed {
// NB: We're searching backwards.
query_builder.push(" m.id < ");
sort_ascending = !sort_ascending;
} else {
query_builder.push(" m.id > ");
}
} else {
if cursor_is_reversed {
// NB: We're searching backwards.
query_builder.push(" m.id > ");
sort_ascending = !sort_ascending;
} else {
query_builder.push(" m.id < ");
}
}
query_builder.push_bind(offset as i64);
}
query_builder
}
async fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<FeaturedWeight> {
let weights: Vec<FeaturedWeight> = dataset
.into_iter()
.map(|weight: RawWeightJoinUser| {
FeaturedWeight {
token: weight.token,
title: weight.title,
weights_type: weight.weights_type,
weights_category: weight.weights_category,
maybe_ietf_language_tag: weight.maybe_ietf_language_tag,
maybe_ietf_primary_language_subtag: weight.maybe_ietf_primary_language_subtag,
maybe_creator_user_token: weight.maybe_creator_user_token,
maybe_creator_username: weight.maybe_creator_username,
maybe_creator_display_name: weight.maybe_creator_display_name,
maybe_creator_email_gravatar_hash: weight.maybe_creator_email_gravatar_hash,
creator_ip_address: weight.creator_ip_address,
creator_set_visibility: weight.creator_set_visibility,
maybe_last_update_user_token: weight.maybe_last_update_user_token,
original_download_url: weight.original_download_url,
original_filename: weight.original_filename,
file_size_bytes: weight.file_size_bytes,
file_checksum_sha2: weight.file_checksum_sha2,
public_bucket_hash: weight.public_bucket_hash,
maybe_public_bucket_prefix: weight.maybe_public_bucket_prefix,
maybe_public_bucket_extension: weight.maybe_public_bucket_extension,
maybe_cover_image_public_bucket_hash: weight.maybe_cover_image_public_bucket_hash,
maybe_cover_image_public_bucket_prefix: weight.maybe_cover_image_public_bucket_prefix,
maybe_cover_image_public_bucket_extension: weight.maybe_cover_image_public_bucket_extension,
maybe_ratings_positive_count: weight.maybe_ratings_positive_count,
maybe_ratings_negative_count: weight.maybe_ratings_negative_count,
maybe_bookmark_count: weight.maybe_bookmark_count,
cached_usage_count: weight.cached_usage_count,
version: weight.version,
created_at: weight.created_at,
updated_at: weight.updated_at,
user_deleted_at: weight.user_deleted_at,
mod_deleted_at: weight.mod_deleted_at,
}
}).collect();
weights
}
struct RawWeightJoinUser {
id: i64,
token: ModelWeightToken,
weights_type: WeightsType,
weights_category: WeightsCategory,
title: String,
maybe_ietf_language_tag: Option<String>,
maybe_ietf_primary_language_subtag: Option<String>,
// TODO(bt,2023-12-24): These aren't really appropriate for a list endpoint.
// Hopefully we don't break the frontend by omitting these.
//pub description_markdown: String,
//pub description_rendered_html: String,
maybe_creator_user_token: Option<UserToken>,
maybe_creator_username: Option<String>,
maybe_creator_display_name: Option<String>,
maybe_creator_email_gravatar_hash: Option<String>,
creator_ip_address: String,
creator_set_visibility: Visibility,
maybe_last_update_user_token: Option<UserToken>,
original_download_url: Option<String>,
original_filename: Option<String>,
file_size_bytes: i32,
file_checksum_sha2: String,
public_bucket_hash: String,
maybe_public_bucket_prefix: Option<String>,
maybe_public_bucket_extension: Option<String>,
cached_usage_count: u64,
maybe_cover_image_public_bucket_hash: Option<String>,
maybe_cover_image_public_bucket_prefix: Option<String>,
maybe_cover_image_public_bucket_extension: Option<String>,
maybe_ratings_positive_count: Option<u32>,
maybe_ratings_negative_count: Option<u32>,
maybe_bookmark_count: Option<u32>,
version: i32,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
user_deleted_at: Option<DateTime<Utc>>,
mod_deleted_at: Option<DateTime<Utc>>,
}
impl FromRow<'_, MySqlRow> for RawWeightJoinUser {
fn from_row(row: &MySqlRow) -> Result<Self, sqlx::Error> {
Ok(Self {
id: row.try_get("id")?,
token: row.try_get("token")?,
weights_type: row.try_get("weights_type")?,
weights_category: row.try_get("weights_category")?,
title: row.try_get("title")?,
maybe_ietf_language_tag: row.try_get("maybe_ietf_language_tag")?,
maybe_ietf_primary_language_subtag: row.try_get("maybe_ietf_primary_language_subtag")?,
// TODO(bt,2023-12-24): These aren't really appropriate for a list endpoint.
// Hopefully we don't break the frontend by omitting these.
//maybe_description_markdown: row.try_get("maybe_description_markdown")?,
//maybe_description_rendered_html: row.try_get("maybe_description_rendered_html")?,
maybe_creator_user_token: row.try_get("maybe_creator_user_token")?,
maybe_creator_username: row.try_get("maybe_creator_username")?,
maybe_creator_display_name: row.try_get("maybe_creator_display_name")?,
maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?,
creator_ip_address: row.try_get("creator_ip_address")?,
creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
maybe_last_update_user_token: row.try_get("maybe_last_update_user_token")?,
original_download_url: row.try_get("original_download_url")?,
original_filename: row.try_get("original_filename")?,
file_size_bytes: row.try_get("file_size_bytes")?,
file_checksum_sha2: row.try_get("file_checksum_sha2")?,
public_bucket_hash: row.try_get("public_bucket_hash")?,
maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
cached_usage_count: row.try_get("cached_usage_count")?,
maybe_cover_image_public_bucket_hash: row.try_get("maybe_cover_image_public_bucket_hash")?,
maybe_cover_image_public_bucket_prefix: row.try_get("maybe_cover_image_public_bucket_prefix")?,
maybe_cover_image_public_bucket_extension: row.try_get("maybe_cover_image_public_bucket_extension")?,
maybe_ratings_positive_count: row.try_get("maybe_ratings_positive_count")?,
maybe_ratings_negative_count: row.try_get("maybe_ratings_negative_count")?,
maybe_bookmark_count: row.try_get("maybe_bookmark_count")?,
version: row.try_get("version")?,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
user_deleted_at: row.try_get("user_deleted_at")?,
mod_deleted_at: row.try_get("mod_deleted_at")?,
})
}
}
@@ -1,6 +1,5 @@
pub mod list_all_model_weight_tokens_for_backfill;
pub mod list_all_model_weight_tokens_for_cached_usage_backfill;
pub mod list_featured_weights;
pub mod list_model_weight_tokens_updated_since;
pub mod list_model_weights_for_elastic_search_backfill_using_cursor;
pub mod list_model_weights_for_text_to_speech;
@@ -1,66 +0,0 @@
use chrono::NaiveDateTime;
use sqlx::MySqlPool;
use errors::AnyhowResult;
use crate::helpers::numeric_converters::try_i64_to_u64_or_min;
#[derive(Serialize, Deserialize, Clone)]
pub struct QueueStatsRow {
pub queue_type: String,
pub pending_job_count: u64,
pub present_time: NaiveDateTime,
}
struct QueueStatsRowInternal {
queue_type: Option<String>, // FIXME: Shouldn't be nullable
pending_job_count: i64,
present_time: NaiveDateTime,
}
// NB: Returning Option<T> is a hack to fit with outer Redis caching.
// I'm only doing this because I'm in a hurry. Don't do this.
pub async fn get_unified_queue_stats(
pool: &MySqlPool,
) -> AnyhowResult<Vec<QueueStatsRow>> {
// NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale
// forever. We currently have no automated means of collecting those jobs.
// NB (2): We query the server timestamp so we can cache the results.
// The frontend can then monotonically adjust the count based on timestamp by ignoring
// past times.
let rows : Vec<QueueStatsRowInternal> = sqlx::query_as!(
QueueStatsRowInternal,
r#"
SELECT
job_type as queue_type,
count(*) as pending_job_count,
NOW() as present_time
FROM (
SELECT
token,
job_type
FROM generic_inference_jobs
WHERE status IN ("pending", "attempt_failed")
AND job_type IS NOT NULL
UNION
SELECT
token,
job_type
FROM generic_inference_jobs
WHERE status IN ("started")
AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
AND job_type IS NOT NULL
) as generic_inner
GROUP BY queue_type
"#,
)
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(|row|
QueueStatsRow {
queue_type: row.queue_type.unwrap_or_else(|| "unknown".to_string()),
pending_job_count: try_i64_to_u64_or_min(row.pending_job_count),
present_time: row.present_time,
}).collect::<Vec<_>>())
}
@@ -1,3 +1,2 @@
pub mod get_on_prem_worker_stats;
pub mod get_unified_queue_stats;
pub mod get_voice_count_stats;
pub mod get_voice_count_stats;
@@ -1,7 +1,6 @@
pub mod stats;
pub mod tts_category_assignments;
pub mod tts_download_jobs;
pub mod tts_inference_jobs;
pub mod tts_model_upload_jobs;
pub mod tts_models;
pub mod tts_results;
pub mod tts_results;
@@ -1,89 +0,0 @@
use anyhow::anyhow;
use log::warn;
use sqlx::MySql;
use sqlx::pool::PoolConnection;
use errors::AnyhowResult;
#[derive(Serialize, Clone)]
pub struct TtsLeaderboardRecordForList {
pub username: String,
pub display_name: String,
pub gravatar_hash: String,
pub creator_user_token: String,
pub uploaded_count: i64,
}
#[derive(Serialize)]
struct TtsLeaderboardRecordForListRaw {
pub username: String,
pub display_name: String,
pub gravatar_hash: String,
pub creator_user_token: String,
pub uploaded_count: i64,
}
// NB: This may need to become an offline query if it is expensive.
pub async fn calculate_tts_model_leaderboard(
mysql_pool: &mut PoolConnection<MySql>
) -> AnyhowResult<Vec<TtsLeaderboardRecordForList>> {
// NB(1): We're not requiring "is_public_listing_approved IS TRUE" for TTS models
// This is the opposite for W2L templates (for now at least)
// NB(2): We're excluding a few users (eg. Vegito's alternate account)
// U:DRB1SKEMKHT1J - https://fakeyou.com/profile/bluefusion/uploaded-tts-models
let maybe_results = sqlx::query_as!(
TtsLeaderboardRecordForListRaw,
r#"
SELECT
users.username,
users.display_name,
users.email_gravatar_hash as gravatar_hash,
creator_user_token,
count(*) as uploaded_count
FROM tts_models
JOIN users
ON
users.token = creator_user_token
WHERE
users.is_banned IS FALSE
AND users.token NOT IN ("U:DRB1SKEMKHT1J")
AND tts_models.user_deleted_at IS NULL
AND tts_models.mod_deleted_at IS NULL
GROUP BY creator_user_token
ORDER BY uploaded_count desc
LIMIT 25;
"#,
)
.fetch_all(&mut **mysql_pool)
.await;
let results : Vec<TtsLeaderboardRecordForListRaw> = match maybe_results {
Ok(results) => results,
Err(err) => {
warn!("Error: {:?}", err);
match err {
sqlx::Error::RowNotFound => {
return Ok(Vec::new());
},
_ => {
warn!("tts leaderboard query error: {:?}", err);
return Err(anyhow!("tts leaderboard query error"));
}
}
}
};
Ok(results.into_iter()
.map(|result| {
TtsLeaderboardRecordForList {
username: result.username.to_string(),
display_name: result.display_name.to_string(),
gravatar_hash: result.gravatar_hash.to_string(),
creator_user_token: result.creator_user_token.to_string(),
uploaded_count: result.uploaded_count,
}
})
.collect::<Vec<TtsLeaderboardRecordForList>>())
}
@@ -1 +0,0 @@
pub mod calculate_tts_model_leaderboard;
@@ -1,279 +0,0 @@
use chrono::{DateTime, Utc};
use sqlx::{MySql, MySqlPool};
use sqlx::pool::PoolConnection;
use errors::AnyhowResult;
use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool};
/// List of assigned categories for a TTS model
/// This is *NOT* paginated, as that defeats the purpose of having a category system.
#[derive(Serialize)]
pub struct AssignedCategoryList {
pub categories: Vec<AssignedCategory>,
}
#[derive(Serialize)]
pub struct AssignedCategory {
pub category_token: String,
pub model_type: String, // TODO: ENUM
pub maybe_super_category_token: Option<String>,
pub can_directly_have_models: bool,
pub can_have_subcategories: bool,
pub can_only_mods_apply: bool,
pub name: String,
pub maybe_dropdown_name: Option<String>,
pub category_creator_user_token: Option<String>,
pub category_creator_username: Option<String>,
pub category_creator_display_name: Option<String>,
pub category_creator_gravatar_hash: Option<String>,
// Moderator fields
pub is_mod_approved: Option<bool>,
pub maybe_mod_comments: Option<String>,
//pub maybe_mod_user_token: Option<String>,
//pub maybe_mod_username: Option<String>,
//pub maybe_mod_display_name: Option<String>,
pub category_created_at: DateTime<Utc>,
pub category_updated_at: DateTime<Utc>,
pub category_deleted_at: Option<DateTime<Utc>>,
pub category_assignment_created_at: DateTime<Utc>,
pub category_assignment_updated_at: DateTime<Utc>,
pub category_assignment_deleted_at: Option<DateTime<Utc>>,
}
/// Query Builder for listing categories
/// These are very difficult queries, so the builder helps for
/// testability, construction, and correctness.
pub struct ListAssignedTtsCategoriesQueryBuilder {
// REQUIRED.
scope_tts_model_token: String,
// Sometimes "can_directly_have_models" categories change state.
// Mods will want to see invalid uses.
show_invalid_model_not_allowed_categories: bool,
// The public lists won't include unapproved, but mod views
// will want to see them.
show_unapproved: bool,
// The public lists won't include deleted models, but mod views
// will want to see them.
show_deleted: bool,
}
impl ListAssignedTtsCategoriesQueryBuilder {
pub fn for_model_token(model_token: &str) -> Self {
Self {
scope_tts_model_token: model_token.to_string(),
show_invalid_model_not_allowed_categories: false,
show_unapproved: false,
show_deleted: false,
}
}
pub fn show_invalid_model_not_allowed_categories(mut self, show_invalid_model_not_allowed_categories: bool) -> Self {
self.show_invalid_model_not_allowed_categories = show_invalid_model_not_allowed_categories;
self
}
pub fn show_unapproved(mut self, show_unapproved: bool) -> Self {
self.show_unapproved = show_unapproved;
self
}
pub fn show_deleted(mut self, show_deleted: bool) -> Self {
self.show_deleted = show_deleted;
self
}
/// Perform the query based on the set predicates.
#[deprecated = "Use the PoolConnection<MySql> method instead of the MySqlPool one."]
pub async fn perform_query(
&self,
mysql_pool: &MySqlPool
) -> AnyhowResult<AssignedCategoryList> {
let mut mysql_connection = mysql_pool.acquire().await?;
self.perform_query_by_connection(&mut mysql_connection).await
}
/// Perform the query based on the set predicates.
pub async fn perform_query_by_connection(
&self,
mysql_connection: &mut PoolConnection<MySql>
) -> AnyhowResult<AssignedCategoryList> {
let internal_results = self.perform_internal_query(mysql_connection).await?;
let categories = internal_results
.into_iter()
.map(|c| {
AssignedCategory {
category_token: c.category_token,
model_type: c.model_type,
maybe_super_category_token: c.maybe_super_category_token,
can_directly_have_models: i8_to_bool(c.can_directly_have_models),
can_have_subcategories: i8_to_bool(c.can_have_subcategories),
can_only_mods_apply: i8_to_bool(c.can_only_mods_apply),
name: c.name,
maybe_dropdown_name: c.maybe_dropdown_name,
category_creator_user_token: c.category_creator_user_token,
category_creator_username: c.category_creator_username,
category_creator_display_name: c.category_creator_display_name,
category_creator_gravatar_hash: c.category_creator_gravatar_hash,
is_mod_approved: nullable_i8_to_optional_bool(c.is_mod_approved),
maybe_mod_comments: c.maybe_mod_comments,
category_created_at: c.category_created_at,
category_updated_at: c.category_updated_at,
category_deleted_at: c.category_deleted_at,
category_assignment_created_at: c.category_assignment_created_at,
category_assignment_updated_at: c.category_assignment_updated_at,
category_assignment_deleted_at: c.category_assignment_deleted_at,
}
})
.collect::<Vec<AssignedCategory>>();
Ok(AssignedCategoryList {
categories,
})
}
/// Perform the query based on the set predicates.
async fn perform_internal_query(
&self,
mysql_connection: &mut PoolConnection<MySql>
) -> AnyhowResult<Vec<RawInternalCategoryRecord>> {
let query = self.build_query_string();
let mut query = sqlx::query_as::<_, RawInternalCategoryRecord>(&query);
// NB: The following bindings must match the order of the query builder !!
// NB: Model token is always used for scoping.
query = query.bind(&self.scope_tts_model_token);
let results = query.fetch_all(&mut **mysql_connection)
.await?;
Ok(results)
}
fn build_query_string(&self) -> String {
// TODO: I haven't figured out how to get field name disambiguation and type coercion working here.
// (1) tts_results.creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`,
// Query error: no column found for name: creator_set_visibility
// (2) creator_set_visibility `creator_set_visibility: crate::database::enums::record_visibility::RecordVisibility`,
// Column 'creator_set_visibility' in field list is ambiguous
// TODO/NB: Unfortunately SQLx can't statically typecheck this query
let mut query = r#"
SELECT
model_categories.token as category_token,
model_categories.model_type,
model_categories.maybe_super_category_token,
model_categories.can_directly_have_models,
model_categories.can_have_subcategories,
model_categories.can_only_mods_apply,
model_categories.name,
model_categories.maybe_dropdown_name,
users.token as category_creator_user_token,
users.username as category_creator_username,
users.display_name as category_creator_display_name,
users.email_gravatar_hash as category_creator_gravatar_hash,
model_categories.is_mod_approved,
model_categories.maybe_mod_comments,
model_categories.created_at as category_created_at,
model_categories.updated_at as category_updated_at,
model_categories.deleted_at as category_deleted_at,
tts_category_assignments.created_at as category_assignment_created_at,
tts_category_assignments.updated_at as category_assignment_updated_at,
tts_category_assignments.deleted_at as category_assignment_deleted_at
FROM model_categories
JOIN tts_category_assignments
ON tts_category_assignments.category_token = model_categories.token
LEFT OUTER JOIN users
ON model_categories.creator_user_token = users.token
"#.to_string();
query.push_str(&self.build_predicates());
query
}
pub fn build_predicates(&self) -> String {
let mut query = "".to_string();
// Scoping to tts model is required.
query.push_str(" WHERE tts_category_assignments.model_token = ?");
query.push_str(" AND model_categories.model_type = 'tts'");
// We also don't care to ever show soft-deleted assignments
query.push_str(" AND tts_category_assignments.deleted_at IS NULL");
if !self.show_invalid_model_not_allowed_categories {
query.push_str(" AND model_categories.can_directly_have_models IS TRUE");
}
if !self.show_unapproved {
query.push_str(" AND model_categories.is_mod_approved IS TRUE");
}
if !self.show_deleted {
query.push_str(" AND model_categories.deleted_at IS NULL");
}
query
}
}
#[derive(sqlx::FromRow)]
pub struct RawInternalCategoryRecord {
pub category_token: String,
pub model_type: String, // TODO: ENUM
pub maybe_super_category_token: Option<String>,
pub can_directly_have_models: i8,
pub can_have_subcategories: i8,
pub can_only_mods_apply: i8,
pub name: String,
pub maybe_dropdown_name: Option<String>,
pub category_creator_user_token: Option<String>,
pub category_creator_username: Option<String>,
pub category_creator_display_name: Option<String>,
pub category_creator_gravatar_hash: Option<String>,
// Moderator fields
pub is_mod_approved: Option<i8>,
pub maybe_mod_comments: Option<String>,
pub category_created_at: DateTime<Utc>,
pub category_updated_at: DateTime<Utc>,
pub category_deleted_at: Option<DateTime<Utc>>,
pub category_assignment_created_at: DateTime<Utc>,
pub category_assignment_updated_at: DateTime<Utc>,
pub category_assignment_deleted_at: Option<DateTime<Utc>>,
}
#[cfg(test)]
mod tests {
}
@@ -1,2 +1 @@
pub mod fetch_and_build_tts_model_category_map;
pub mod list_assigned_tts_categories_query_builder;
@@ -1,53 +0,0 @@
use chrono::NaiveDateTime;
use sqlx::MySqlPool;
use errors::AnyhowResult;
use crate::helpers::numeric_converters::try_i64_to_u64_or_min;
#[derive(Clone)]
pub struct TtsQueueLengthResult {
pub record_count: u64,
pub present_time: NaiveDateTime,
}
struct QueryResultInternal {
record_count: i64,
present_time: NaiveDateTime,
}
pub async fn get_pending_tts_inference_job_count(
pool: &MySqlPool,
) -> AnyhowResult<TtsQueueLengthResult> {
// NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale
// forever. We currently have no automated means of collecting those jobs.
// NB (2): We query the server timestamp so we can cache the results.
// The frontend can then monotonically adjust the count based on timestamp by ignoring
// past times.
let result : QueryResultInternal = sqlx::query_as!(
QueryResultInternal,
r#"
SELECT
COUNT(distinct token) as record_count,
NOW() as present_time
FROM
(
SELECT token
FROM tts_inference_jobs
WHERE status = "started"
AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
UNION
SELECT token
FROM tts_inference_jobs
WHERE status IN ("pending", "attempt_failed")
) as t
"#,
)
.fetch_one(pool)
.await?;
Ok(TtsQueueLengthResult {
record_count: try_i64_to_u64_or_min(result.record_count),
present_time: result.present_time,
})
}
@@ -1,74 +0,0 @@
use anyhow::anyhow;
use chrono::{DateTime, Utc};
use sqlx::MySqlPool;
use errors::AnyhowResult;
#[derive(Serialize)]
pub struct TtsInferenceJobStatusRecord {
pub job_token: String,
pub status: String,
pub attempt_count: i32,
pub maybe_result_token: Option<String>,
pub maybe_public_bucket_wav_audio_path: Option<String>,
pub model_token: String,
pub tts_model_type: String,
pub title: String, // Name of the TTS model
pub raw_inference_text: String, // User text
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
pub async fn get_tts_inference_job_status(
tts_inference_job_token: &str,
mysql_pool: &MySqlPool
) -> AnyhowResult<Option<TtsInferenceJobStatusRecord>> {
// NB: Lookup failure is Err(RowNotFound).
// NB: Since this is publicly exposed, we don't query sensitive data.
let maybe_status = sqlx::query_as!(
TtsInferenceJobStatusRecord,
r#"
SELECT
jobs.token as job_token,
jobs.status,
jobs.attempt_count,
jobs.on_success_result_token as maybe_result_token,
results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path,
jobs.model_token,
tts.tts_model_type,
tts.title,
jobs.raw_inference_text,
jobs.created_at,
jobs.updated_at
FROM tts_inference_jobs as jobs
JOIN tts_models as tts
ON tts.token = jobs.model_token
LEFT OUTER JOIN tts_results as results
ON jobs.on_success_result_token = results.token
WHERE jobs.token = ?
"#,
tts_inference_job_token
)
.fetch_one(mysql_pool)
.await; // TODO: This will return error if it doesn't exist
match maybe_status {
Ok(record) => Ok(Some(record)),
Err(err) => match err {
sqlx::Error::RowNotFound => Ok(None),
_ => Err(anyhow!("tts job query error: {:?}", err)),
}
}
}
@@ -1,7 +1,5 @@
pub mod _keys;
pub mod get_pending_tts_inference_job_count;
pub mod get_pending_tts_inference_job_detailed_stats;
pub mod get_tts_inference_job_status;
pub mod insert_tts_inference_job;
pub mod kill_tts_inference_jobs;
pub mod list_available_tts_inference_jobs;
@@ -1,82 +0,0 @@
use anyhow::anyhow;
use log::warn;
use sqlx::error::Error::Database;
use sqlx::MySqlPool;
use errors::AnyhowResult;
use tokens::tokens::tts_model_upload_jobs::TtsModelUploadJobToken;
pub struct InsertTtsModelUploadJobArgs<'a> {
pub uuid: &'a str,
pub creator_user_token: &'a str,
pub creator_ip_address: &'a str,
pub creator_set_visibility: &'a str,
pub title: &'a str,
pub tts_model_type: &'a str,
pub download_url: &'a str,
pub mysql_pool: &'a MySqlPool,
}
// NB: Returns job token
pub async fn insert_tts_model_upload_job(args: InsertTtsModelUploadJobArgs<'_>) -> AnyhowResult<String> {
// This token is returned to the client.
let job_token = TtsModelUploadJobToken::generate().to_string();
let query_result = sqlx::query!(
r#"
INSERT INTO tts_model_upload_jobs
SET
token = ?,
uuid_idempotency_token = ?,
creator_user_token = ?,
creator_ip_address = ?,
creator_set_visibility = ?,
title = ?,
tts_model_type = ?,
download_url = ?,
status = "pending"
"#,
job_token.to_string(),
args.uuid,
args.creator_user_token,
args.creator_ip_address,
args.creator_set_visibility,
args.title,
args.tts_model_type,
args.download_url,
)
.execute(args.mysql_pool)
.await;
let _record_id = match query_result {
Ok(res) => {
res.last_insert_id()
},
Err(err) => {
warn!("New tts model upload creation DB error: {:?}", err);
// NB: SQLSTATE[23000]: Integrity constraint violation
// NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access)
match err {
Database(ref err) => {
let _maybe_code = err.code().map(|c| c.into_owned());
/*match maybe_code.as_deref() {
Some("23000") => {
if err.message().contains("username") {
return Err(UsernameTaken);
} else if err.message().contains("email_address") {
return Err(EmailTaken);
}
}
_ => {},
}*/
},
_ => {},
}
return Err(anyhow!("error inserting model upload job record: {:?}", err));
}
};
Ok(job_token)
}
@@ -1,2 +1 @@
pub mod get_tts_model_upload_job_status;
pub mod insert_tts_model_upload_job;
@@ -1,89 +0,0 @@
use sqlx::MySqlPool;
use errors::AnyhowResult;
pub async fn delete_tts_inference_result_as_user(
inference_result_token: &str,
mysql_pool: &MySqlPool
) -> AnyhowResult<()> {
let _r = sqlx::query!(
r#"
UPDATE tts_results
SET
user_deleted_at = CURRENT_TIMESTAMP
WHERE
token = ?
LIMIT 1
"#,
inference_result_token,
)
.execute(mysql_pool)
.await?;
Ok(())
}
pub async fn delete_tts_inference_result_as_mod(
inference_result_token: &str,
mod_user_token: &str,
mysql_pool: &MySqlPool
) -> AnyhowResult<()> {
let _r = sqlx::query!(
r#"
UPDATE tts_results
SET
mod_deleted_at = CURRENT_TIMESTAMP,
maybe_mod_user_token = ?
WHERE
token = ?
LIMIT 1
"#,
mod_user_token,
inference_result_token,
)
.execute(mysql_pool)
.await?;
Ok(())
}
pub async fn undelete_tts_inference_result_as_user(
inference_result_token: &str,
mysql_pool: &MySqlPool
) -> AnyhowResult<()> {
let _r = sqlx::query!(
r#"
UPDATE tts_results
SET
user_deleted_at = NULL
WHERE
token = ?
LIMIT 1
"#,
inference_result_token,
)
.execute(mysql_pool)
.await?;
Ok(())
}
pub async fn undelete_tts_inference_result_as_mod(
inference_result_token: &str,
mod_user_token: &str,
mysql_pool: &MySqlPool
) -> AnyhowResult<()> {
let _r = sqlx::query!(
r#"
UPDATE tts_results
SET
mod_deleted_at = NULL,
maybe_mod_user_token = ?
WHERE
token = ?
LIMIT 1
"#,
mod_user_token,
inference_result_token,
)
.execute(mysql_pool)
.await?;
Ok(())
}
@@ -1,5 +1,4 @@
pub mod delete_tts_result_various_scopes;
pub mod edit_tts_result;
pub mod insert_tts_result;
pub mod list_tts_results_query_builder;
pub mod query_tts_result;
pub mod query_tts_result;
@@ -0,0 +1,78 @@
use std::marker::PhantomData;
use chrono::{DateTime, Utc};
use sqlx::{Executor, MySql};
use tokens::tokens::users::UserToken;
/// One user's aggregated spend within the requested window, joined to their
/// display info. Only attributed (non-null user), production events count.
pub struct TopSpenderForWindow {
pub user_token: UserToken,
pub username: String,
pub display_name: String,
pub email_gravatar_hash: String,
pub gross_spend_usd_cents: u64,
pub refund_usd_cents: u64,
pub net_spend_usd_cents: i64,
pub payment_count: u64,
pub credits_granted: u64,
}
pub struct ListTopSpendersForWindowArgs<'e, 'c, E>
where
E: 'e + Executor<'c, Database = MySql>,
{
/// Inclusive start of the aggregation window.
pub window_start: DateTime<Utc>,
/// Optional namespace filter. `None` aggregates across all namespaces.
pub maybe_payments_namespace: Option<&'e str>,
pub limit: i64,
pub offset: i64,
pub mysql_executor: E,
pub phantom: PhantomData<&'c E>,
}
/// Top spenders since `window_start`, sorted by net spend descending.
/// Aggregates `user_spend_events` directly (exact rolling windows, unlike the
/// daily rollups) — the table is payment-scale, so this stays cheap.
pub async fn list_top_spenders_for_window<'e, 'c: 'e, E>(
args: ListTopSpendersForWindowArgs<'e, 'c, E>,
) -> Result<Vec<TopSpenderForWindow>, sqlx::Error>
where
E: 'e + Executor<'c, Database = MySql>,
{
let rows = sqlx::query_as!(
TopSpenderForWindow,
r#"
SELECT
e.maybe_user_token AS `user_token!: UserToken`,
u.username AS `username!`,
u.display_name AS `display_name!`,
u.email_gravatar_hash AS `email_gravatar_hash!`,
CAST(SUM(CASE WHEN e.amount_usd_cents > 0 THEN e.amount_usd_cents ELSE 0 END) AS UNSIGNED) AS `gross_spend_usd_cents!: u64`,
CAST(SUM(CASE WHEN e.amount_usd_cents < 0 THEN -e.amount_usd_cents ELSE 0 END) AS UNSIGNED) AS `refund_usd_cents!: u64`,
CAST(SUM(e.amount_usd_cents) AS SIGNED) AS `net_spend_usd_cents!: i64`,
CAST(SUM(CASE WHEN e.amount_usd_cents > 0 THEN 1 ELSE 0 END) AS UNSIGNED) AS `payment_count!: u64`,
CAST(COALESCE(SUM(e.maybe_credits_granted), 0) AS UNSIGNED) AS `credits_granted!: u64`
FROM user_spend_events e
JOIN users u ON u.token = e.maybe_user_token
WHERE e.maybe_user_token IS NOT NULL
AND e.is_production = TRUE
AND e.payment_occurred_at >= ?
AND (? IS NULL OR e.payments_namespace = ?)
GROUP BY e.maybe_user_token, u.username, u.display_name, u.email_gravatar_hash
ORDER BY SUM(e.amount_usd_cents) DESC, e.maybe_user_token ASC
LIMIT ? OFFSET ?
"#,
args.window_start,
args.maybe_payments_namespace,
args.maybe_payments_namespace,
args.limit,
args.offset,
)
.fetch_all(args.mysql_executor)
.await?;
Ok(rows)
}
@@ -1,3 +1,4 @@
pub mod list_top_spenders_for_window;
pub mod list_user_spend_events_for_moderation;
pub mod backfill_get_user_token_by_email;
pub mod backfill_list_artcraft_customer_links;
@@ -0,0 +1,21 @@
use tokens::tokens::api_keys::ApiKeyToken;
use tokens::tokens::users::UserToken;
/// A user authenticated by either an API key or a web session cookie.
///
/// The two lookup queries (`get_api_or_web_session_user_by_api_key` and
/// `get_api_or_web_session_user_by_session_token`) select congruent user fields so both can
/// produce this one uniform record.
pub struct ApiOrWebSessionUserRecord {
pub user_token: UserToken,
pub username: String,
pub display_name: String,
pub email_address: String,
pub user_role_slug: String,
pub is_banned: bool,
pub can_ban_users: bool,
/// Only set when the lookup was by API key.
pub maybe_api_key_token: Option<ApiKeyToken>,
}
@@ -0,0 +1,70 @@
use std::marker::PhantomData;
use sqlx::{Executor, MySql};
use artcraft_api_keys::ArtcraftApiKey;
use tokens::tokens::api_keys::ApiKeyToken;
use tokens::tokens::users::UserToken;
use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_bool_default_false};
use crate::queries::users::api_or_web_sessions::api_or_web_session_user_record::ApiOrWebSessionUserRecord;
pub struct GetApiOrWebSessionUserByApiKeyArgs<'e, 'c, E>
where
E: 'e + Executor<'c, Database = MySql>,
{
/// The actual API key secret presented by the client.
pub api_key: &'e ArtcraftApiKey,
pub mysql_executor: E,
pub phantom: PhantomData<&'c E>,
}
/// Look up the user behind an API key secret, selecting the same user fields as
/// `get_api_or_web_session_user_by_session_token` so both auth paths share one record shape.
///
/// Returns `Ok(None)` when no row matches, when the matching API key is soft-deleted, or when the
/// owning user is missing or deleted. The full secret is never echoed back.
pub async fn get_api_or_web_session_user_by_api_key<'e, 'c: 'e, E>(
args: GetApiOrWebSessionUserByApiKeyArgs<'e, 'c, E>,
) -> Result<Option<ApiOrWebSessionUserRecord>, sqlx::Error>
where
E: 'e + Executor<'c, Database = MySql>,
{
let result = sqlx::query!(
r#"
SELECT
users.token as `user_token: UserToken`,
users.username,
users.display_name,
users.email_address,
users.user_role_slug,
users.is_banned,
user_roles.can_ban_users,
api_keys.token as `api_key_token: ApiKeyToken`
FROM api_keys
INNER JOIN users
ON users.token = api_keys.owner_user_token
LEFT OUTER JOIN user_roles
ON users.user_role_slug = user_roles.slug
WHERE api_keys.api_key = ?
AND api_keys.maybe_deleted_at IS NULL
AND users.user_deleted_at IS NULL
AND users.mod_deleted_at IS NULL
LIMIT 1
"#,
args.api_key.as_str_be_careful(),
)
.fetch_optional(args.mysql_executor)
.await?;
Ok(result.map(|r| ApiOrWebSessionUserRecord {
user_token: r.user_token,
username: r.username,
display_name: r.display_name,
email_address: r.email_address,
user_role_slug: r.user_role_slug,
is_banned: i8_to_bool(r.is_banned),
can_ban_users: nullable_i8_to_bool_default_false(r.can_ban_users),
maybe_api_key_token: Some(r.api_key_token),
}))
}
@@ -0,0 +1,67 @@
use std::marker::PhantomData;
use sqlx::{Executor, MySql};
use tokens::tokens::users::UserToken;
use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_bool_default_false};
use crate::queries::users::api_or_web_sessions::api_or_web_session_user_record::ApiOrWebSessionUserRecord;
pub struct GetApiOrWebSessionUserBySessionTokenArgs<'e, 'c, E>
where
E: 'e + Executor<'c, Database = MySql>,
{
/// The session token decoded from the session cookie.
pub session_token: &'e str,
pub mysql_executor: E,
pub phantom: PhantomData<&'c E>,
}
/// Look up the user behind a web session token, selecting the same user fields as
/// `get_api_or_web_session_user_by_api_key` so both auth paths share one record shape.
///
/// Returns `Ok(None)` when no row matches, when the session is deleted, or when the owning user
/// is missing or deleted.
pub async fn get_api_or_web_session_user_by_session_token<'e, 'c: 'e, E>(
args: GetApiOrWebSessionUserBySessionTokenArgs<'e, 'c, E>,
) -> Result<Option<ApiOrWebSessionUserRecord>, sqlx::Error>
where
E: 'e + Executor<'c, Database = MySql>,
{
let result = sqlx::query!(
r#"
SELECT
users.token as `user_token: UserToken`,
users.username,
users.display_name,
users.email_address,
users.user_role_slug,
users.is_banned,
user_roles.can_ban_users
FROM user_sessions
INNER JOIN users
ON users.token = user_sessions.user_token
LEFT OUTER JOIN user_roles
ON users.user_role_slug = user_roles.slug
WHERE user_sessions.token = ?
AND user_sessions.deleted_at IS NULL
AND users.user_deleted_at IS NULL
AND users.mod_deleted_at IS NULL
LIMIT 1
"#,
args.session_token,
)
.fetch_optional(args.mysql_executor)
.await?;
Ok(result.map(|r| ApiOrWebSessionUserRecord {
user_token: r.user_token,
username: r.username,
display_name: r.display_name,
email_address: r.email_address,
user_role_slug: r.user_role_slug,
is_banned: i8_to_bool(r.is_banned),
can_ban_users: nullable_i8_to_bool_default_false(r.can_ban_users),
maybe_api_key_token: None,
}))
}
@@ -0,0 +1,3 @@
pub mod api_or_web_session_user_record;
pub mod get_api_or_web_session_user_by_api_key;
pub mod get_api_or_web_session_user_by_session_token;
@@ -1,3 +1,4 @@
pub mod api_or_web_sessions;
pub mod user;
pub mod user_badges;
pub mod user_bookmarks;
@@ -1,237 +0,0 @@
use anyhow::anyhow;
use chrono::Utc;
use sqlx::MySqlPool;
use errors::AnyhowResult;
use crate::helpers::boolean_converters::i8_to_bool;
/// Lookup by voice clone request token
pub async fn get_voice_clone_request_by_token(
voice_clone_request_token: &str,
pool: &MySqlPool,
) -> AnyhowResult<Option<VoiceCloneRequest>> {
internal_voice_clone_request_by_token(voice_clone_request_token, pool).await
}
/// Lookup by user
pub async fn get_voice_clone_request_by_user_token(
user_token: &str,
pool: &MySqlPool,
) -> AnyhowResult<Option<VoiceCloneRequest>> {
internal_voice_clone_request_by_user_token(user_token, pool).await
}
/// Used for both *BY TOKEN* and for *BY USER* lookups.
/// The user field should be optional for one and not for the other, but code reuse is more
/// important and this is a quick / hasty feature.
#[derive(Debug)]
pub struct VoiceCloneRequest {
pub token: String,
// Contact
pub maybe_user_token: Option<String>,
pub email_address: String,
pub discord_username: String,
// Visibility
pub is_for_private_use: bool,
pub is_for_public_use: bool,
// Use
pub is_for_studio: bool,
pub is_for_twitch_tts: bool,
pub is_for_api_use: bool,
pub is_for_music: bool,
pub is_for_games: bool,
pub is_for_other: bool,
pub optional_notes_on_use: Option<String>,
// Subject/Ownership
pub is_own_voice: bool,
pub is_third_party_voice: bool,
pub optional_notes_on_subject: Option<String>,
// Equipment
pub has_clean_audio_recordings: bool,
pub has_good_microphone: bool,
// Comments
pub optional_questions: Option<String>,
pub optional_extra_comments: Option<String>,
pub created_at: chrono::DateTime<Utc>,
pub updated_at: chrono::DateTime<Utc>,
}
async fn internal_voice_clone_request_by_token(
voice_clone_request_token: &str,
pool: &MySqlPool,
) -> AnyhowResult<Option<VoiceCloneRequest>> {
let maybe_record = sqlx::query_as!(
VoiceCloneRequestInternal,
r#"
SELECT
token,
maybe_user_token,
email_address,
discord_username,
is_for_private_use,
is_for_public_use,
is_for_studio,
is_for_twitch_tts,
is_for_api_use,
is_for_music,
is_for_games,
is_for_other,
optional_notes_on_use,
is_own_voice,
is_third_party_voice,
optional_notes_on_subject,
has_clean_audio_recordings,
has_good_microphone,
optional_questions,
optional_extra_comments,
created_at,
updated_at
FROM voice_clone_requests
WHERE
token = ?
"#,
voice_clone_request_token,
)
.fetch_one(pool)
.await;
handle_result(maybe_record)
}
async fn internal_voice_clone_request_by_user_token(
user_token: &str,
pool: &MySqlPool,
) -> AnyhowResult<Option<VoiceCloneRequest>> {
let maybe_record = sqlx::query_as!(
VoiceCloneRequestInternal,
r#"
SELECT
token,
maybe_user_token,
email_address,
discord_username,
is_for_private_use,
is_for_public_use,
is_for_studio,
is_for_twitch_tts,
is_for_api_use,
is_for_music,
is_for_games,
is_for_other,
optional_notes_on_use,
is_own_voice,
is_third_party_voice,
optional_notes_on_subject,
has_clean_audio_recordings,
has_good_microphone,
optional_questions,
optional_extra_comments,
created_at,
updated_at
FROM voice_clone_requests
WHERE
maybe_user_token = ?
"#,
user_token,
)
.fetch_one(pool)
.await;
handle_result(maybe_record)
}
fn handle_result(
maybe_record: Result<VoiceCloneRequestInternal, sqlx::Error>,
) -> AnyhowResult<Option<VoiceCloneRequest>> {
let record : VoiceCloneRequestInternal = match maybe_record {
Ok(record) => record,
Err(sqlx::Error::RowNotFound) => return Ok(None),
Err(ref err) => return Err(anyhow!("database query error: {:?}", err)),
};
Ok(Some(VoiceCloneRequest {
token: record.token,
maybe_user_token: record.maybe_user_token,
email_address: record.email_address,
discord_username: record.discord_username,
is_for_private_use: i8_to_bool(record.is_for_private_use),
is_for_public_use: i8_to_bool(record.is_for_public_use),
is_for_studio: i8_to_bool(record.is_for_studio),
is_for_twitch_tts: i8_to_bool(record.is_for_twitch_tts),
is_for_api_use: i8_to_bool(record.is_for_api_use),
is_for_music: i8_to_bool(record.is_for_music),
is_for_games: i8_to_bool(record.is_for_games),
is_for_other: i8_to_bool(record.is_for_other),
optional_notes_on_use: record.optional_notes_on_use,
is_own_voice: i8_to_bool(record.is_own_voice),
is_third_party_voice: i8_to_bool(record.is_third_party_voice),
optional_notes_on_subject: record.optional_notes_on_subject,
has_clean_audio_recordings: i8_to_bool(record.has_clean_audio_recordings),
has_good_microphone: i8_to_bool(record.has_good_microphone),
optional_questions: record.optional_questions,
optional_extra_comments: record.optional_extra_comments,
created_at: record.created_at,
updated_at: record.updated_at,
}))
}
#[derive(Debug)]
struct VoiceCloneRequestInternal {
pub token: String,
// Contact
pub maybe_user_token: Option<String>,
pub email_address: String,
pub discord_username: String,
// Visibility
pub is_for_private_use: i8,
pub is_for_public_use: i8,
// Use
pub is_for_studio: i8,
pub is_for_twitch_tts: i8,
pub is_for_api_use: i8,
pub is_for_music: i8,
pub is_for_games: i8,
pub is_for_other: i8,
pub optional_notes_on_use: Option<String>,
// Subject/Ownership
pub is_own_voice: i8,
pub is_third_party_voice: i8,
pub optional_notes_on_subject: Option<String>,
// Equipment
pub has_clean_audio_recordings: i8,
pub has_good_microphone: i8,
// Comments
pub optional_questions: Option<String>,
pub optional_extra_comments: Option<String>,
pub created_at: chrono::DateTime<Utc>,
pub updated_at: chrono::DateTime<Utc>,
}
@@ -1,125 +0,0 @@
use anyhow::anyhow;
use sqlx::MySqlPool;
use errors::AnyhowResult;
use tokens::tokens::voice_clone_requests::VoiceCloneRequestToken;
pub struct InsertVoiceCloneRequestBuilder {
pub uuid_idempotency_token: String,
// Contact
pub maybe_user_token: Option<String>,
pub email_address: String,
pub discord_username: String,
// Visibility
pub is_for_private_use: bool,
pub is_for_public_use: bool,
// Use
pub is_for_studio: bool,
pub is_for_twitch_tts: bool,
pub is_for_api_use: bool,
pub is_for_music: bool,
pub is_for_games: bool,
pub is_for_other: bool,
pub optional_notes_on_use: Option<String>,
// Subject/Ownership
pub is_own_voice: bool,
pub is_third_party_voice: bool,
pub optional_notes_on_subject: Option<String>,
// Equipment
pub has_clean_audio_recordings: bool,
pub has_good_microphone: bool,
// Comments
pub optional_questions: Option<String>,
pub optional_extra_comments: Option<String>,
pub ip_address_creation: String,
}
impl InsertVoiceCloneRequestBuilder {
/// Returns the newly generated token.
pub async fn insert(&self, mysql_pool: &MySqlPool) -> AnyhowResult<String> {
let token = VoiceCloneRequestToken::generate().to_string();
let query = sqlx::query!(
r#"
INSERT INTO voice_clone_requests
SET
token = ?,
uuid_idempotency_token = ?,
maybe_user_token = ?,
email_address = ?,
discord_username = ?,
is_for_private_use = ?,
is_for_public_use = ?,
is_for_studio = ?,
is_for_twitch_tts = ?,
is_for_api_use = ?,
is_for_music = ?,
is_for_games = ?,
is_for_other = ?,
optional_notes_on_use = ?,
is_own_voice = ?,
is_third_party_voice = ?,
optional_notes_on_subject = ?,
has_clean_audio_recordings = ?,
has_good_microphone = ?,
optional_questions = ?,
optional_extra_comments = ?,
ip_address_creation = ?
"#,
&token,
&self.uuid_idempotency_token,
&self.maybe_user_token,
&self.email_address,
&self.discord_username,
&self.is_for_private_use,
&self.is_for_public_use,
&self.is_for_studio,
&self.is_for_twitch_tts,
&self.is_for_api_use,
&self.is_for_music,
&self.is_for_games,
&self.is_for_other,
&self.optional_notes_on_use,
&self.is_own_voice,
&self.is_third_party_voice,
&self.optional_notes_on_subject,
&self.has_clean_audio_recordings,
&self.has_good_microphone,
&self.optional_questions,
&self.optional_extra_comments,
&self.ip_address_creation,
);
let query_result = query.execute(mysql_pool).await;
let _record_id = match query_result {
Ok(res) => res.last_insert_id(),
Err(err) => return Err(anyhow!("Error creating Voice Clone Request: {:?}", err)),
};
Ok(token)
}
}
@@ -1,2 +0,0 @@
pub mod get_voice_clone_request;
pub mod insert_voice_clone_requests_builder;
@@ -17,10 +17,6 @@ impl RedisCacheKeys {
format!("get_tts_inf2:{}", model_token)
}
pub fn get_unified_queue_stats_endpoint() -> &'static str {
"q_stats_2" // NB: Schema changed, hence _2
}
pub fn session_record_light(session_token: &str) -> String {
format!("ses_light:{}", session_token)
}
@@ -0,0 +1,151 @@
use std::collections::BTreeSet;
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
use crate::error::enum_error::EnumError;
/// Maximum serialized string length for database storage.
pub const MAX_LENGTH: usize = 16;
/// Used in the `debug_logs` table in a `VARCHAR(16)` field (`maybe_log_level`).
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum DebugLogLevel {
Info,
Warn,
Error,
Debug,
Trace,
}
impl_enum_display_and_debug_using_to_str!(DebugLogLevel);
impl_mysql_enum_coders!(DebugLogLevel);
impl_mysql_from_row!(DebugLogLevel);
impl DebugLogLevel {
pub fn to_str(&self) -> &'static str {
match self {
Self::Info => "info",
Self::Warn => "warn",
Self::Error => "error",
Self::Debug => "debug",
Self::Trace => "trace",
}
}
pub fn from_str(value: &str) -> Result<Self, EnumError> {
match value {
"info" => Ok(Self::Info),
"warn" => Ok(Self::Warn),
"error" => Ok(Self::Error),
"debug" => Ok(Self::Debug),
"trace" => Ok(Self::Trace),
_ => Err(EnumError::CouldNotConvertFromString(value.to_string())),
}
}
pub fn all_variants() -> BTreeSet<Self> {
BTreeSet::from([
Self::Info,
Self::Warn,
Self::Error,
Self::Debug,
Self::Trace,
])
}
}
#[cfg(test)]
mod tests {
use crate::by_table::debug_logs::debug_log_level::DebugLogLevel;
use crate::by_table::debug_logs::debug_log_level::MAX_LENGTH;
use crate::test_helpers::assert_serialization;
mod explicit_checks {
use super::*;
use crate::error::enum_error::EnumError;
#[test]
fn test_serialization() {
assert_serialization(DebugLogLevel::Info, "info");
assert_serialization(DebugLogLevel::Warn, "warn");
assert_serialization(DebugLogLevel::Error, "error");
assert_serialization(DebugLogLevel::Debug, "debug");
assert_serialization(DebugLogLevel::Trace, "trace");
}
#[test]
fn to_str() {
assert_eq!(DebugLogLevel::Info.to_str(), "info");
assert_eq!(DebugLogLevel::Warn.to_str(), "warn");
assert_eq!(DebugLogLevel::Error.to_str(), "error");
assert_eq!(DebugLogLevel::Debug.to_str(), "debug");
assert_eq!(DebugLogLevel::Trace.to_str(), "trace");
}
#[test]
fn from_str() {
assert_eq!(DebugLogLevel::from_str("info").unwrap(), DebugLogLevel::Info);
assert_eq!(DebugLogLevel::from_str("warn").unwrap(), DebugLogLevel::Warn);
assert_eq!(DebugLogLevel::from_str("error").unwrap(), DebugLogLevel::Error);
assert_eq!(DebugLogLevel::from_str("debug").unwrap(), DebugLogLevel::Debug);
assert_eq!(DebugLogLevel::from_str("trace").unwrap(), DebugLogLevel::Trace);
}
#[test]
fn from_str_err() {
let result = DebugLogLevel::from_str("invalid");
assert!(result.is_err());
if let Err(EnumError::CouldNotConvertFromString(value)) = result {
assert_eq!(value, "invalid");
} else {
panic!("Expected EnumError::CouldNotConvertFromString");
}
}
#[test]
fn all_variants() {
let mut variants = DebugLogLevel::all_variants();
assert_eq!(variants.len(), 5);
assert_eq!(variants.pop_first(), Some(DebugLogLevel::Info));
assert_eq!(variants.pop_first(), Some(DebugLogLevel::Warn));
assert_eq!(variants.pop_first(), Some(DebugLogLevel::Error));
assert_eq!(variants.pop_first(), Some(DebugLogLevel::Debug));
assert_eq!(variants.pop_first(), Some(DebugLogLevel::Trace));
assert_eq!(variants.pop_first(), None);
}
}
mod mechanical_checks {
use super::*;
#[test]
fn variant_length() {
use strum::IntoEnumIterator;
assert_eq!(DebugLogLevel::all_variants().len(), DebugLogLevel::iter().len());
}
#[test]
fn round_trip() {
for variant in DebugLogLevel::all_variants() {
assert_eq!(variant, DebugLogLevel::from_str(variant.to_str()).unwrap());
assert_eq!(variant, DebugLogLevel::from_str(&format!("{}", variant)).unwrap());
assert_eq!(variant, DebugLogLevel::from_str(&format!("{:?}", variant)).unwrap());
}
}
#[test]
fn max_length() {
for variant in DebugLogLevel::all_variants() {
assert!(variant.to_str().len() <= MAX_LENGTH);
}
}
}
}
@@ -18,7 +18,16 @@ pub const MAX_LENGTH: usize = 24;
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum DebugLogType {
// ========== First party ==========
/// A user request, typically the Json<T> actix_web handlers read.
HttpRequest,
/// A backend/pipeline failure while serving a request.
BackendFailure,
// ========== Third party ==========
FalRequest,
KinoviRequest,
GrokApiRequest,
@@ -34,6 +43,7 @@ impl DebugLogType {
pub fn to_str(&self) -> &'static str {
match self {
Self::HttpRequest => "http_request",
Self::BackendFailure => "backend_failure",
Self::FalRequest => "fal_request",
Self::KinoviRequest => "kinovi_request",
Self::GrokApiRequest => "grok_api_request",
@@ -45,6 +55,7 @@ impl DebugLogType {
pub fn from_str(value: &str) -> Result<Self, EnumError> {
match value {
"http_request" => Ok(Self::HttpRequest),
"backend_failure" => Ok(Self::BackendFailure),
"fal_request" => Ok(Self::FalRequest),
"kinovi_request" => Ok(Self::KinoviRequest),
"grok_api_request" => Ok(Self::GrokApiRequest),
@@ -57,6 +68,7 @@ impl DebugLogType {
pub fn all_variants() -> BTreeSet<Self> {
BTreeSet::from([
Self::HttpRequest,
Self::BackendFailure,
Self::FalRequest,
Self::KinoviRequest,
Self::GrokApiRequest,
@@ -79,6 +91,7 @@ mod tests {
#[test]
fn test_serialization() {
assert_serialization(DebugLogType::HttpRequest, "http_request");
assert_serialization(DebugLogType::BackendFailure, "backend_failure");
assert_serialization(DebugLogType::FalRequest, "fal_request");
assert_serialization(DebugLogType::KinoviRequest, "kinovi_request");
assert_serialization(DebugLogType::GrokApiRequest, "grok_api_request");
@@ -89,6 +102,7 @@ mod tests {
#[test]
fn to_str() {
assert_eq!(DebugLogType::HttpRequest.to_str(), "http_request");
assert_eq!(DebugLogType::BackendFailure.to_str(), "backend_failure");
assert_eq!(DebugLogType::FalRequest.to_str(), "fal_request");
assert_eq!(DebugLogType::KinoviRequest.to_str(), "kinovi_request");
assert_eq!(DebugLogType::GrokApiRequest.to_str(), "grok_api_request");
@@ -99,6 +113,7 @@ mod tests {
#[test]
fn from_str() {
assert_eq!(DebugLogType::from_str("http_request").unwrap(), DebugLogType::HttpRequest);
assert_eq!(DebugLogType::from_str("backend_failure").unwrap(), DebugLogType::BackendFailure);
assert_eq!(DebugLogType::from_str("fal_request").unwrap(), DebugLogType::FalRequest);
assert_eq!(DebugLogType::from_str("kinovi_request").unwrap(), DebugLogType::KinoviRequest);
assert_eq!(DebugLogType::from_str("grok_api_request").unwrap(), DebugLogType::GrokApiRequest);
@@ -120,8 +135,9 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = DebugLogType::all_variants();
assert_eq!(variants.len(), 6);
assert_eq!(variants.len(), 7);
assert_eq!(variants.pop_first(), Some(DebugLogType::HttpRequest));
assert_eq!(variants.pop_first(), Some(DebugLogType::BackendFailure));
assert_eq!(variants.pop_first(), Some(DebugLogType::FalRequest));
assert_eq!(variants.pop_first(), Some(DebugLogType::KinoviRequest));
assert_eq!(variants.pop_first(), Some(DebugLogType::GrokApiRequest));
@@ -1 +1,2 @@
pub mod debug_log_level;
pub mod debug_log_type;
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "trace_id"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[package.metadata.cargo-udeps.ignore]
normal = ["workspace-build-acceleration"]
[lib]
name = "trace_id"
path = "src/lib.rs"
[dependencies]
# Workspace hack (faster compile times)
workspace-build-acceleration.workspace = true
rand = { version = "0.8.5", features = ["std_rng"] }
serde.workspace = true
serde_derive.workspace = true
tokio.workspace = true
+156
View File
@@ -0,0 +1,156 @@
//! trace_id
//!
//! Request-scoped trace identifiers for observability.
//!
//! A `TraceId` is generated for every inbound HTTP request (see the
//! `trace_id_middleware` in `storyteller-web`), stored in a tokio task-local,
//! and emitted with every log line and pager notification produced while
//! serving that request.
use std::fmt;
use serde_derive::{Deserialize, Serialize};
/// The string prefix for trace ids.
pub const TRACE_ID_PREFIX: &str = "trace_";
/// Number of Crockford base32 characters after the prefix.
/// 26 characters × 5 bits = 130 bits of entropy (≥ the 128-bit industry
/// standard for trace ids). Total length: 6 + 26 = 32 characters.
pub const TRACE_ID_ENTROPY_CHARS: usize = 26;
/// Total serialized length of a trace id ("trace_" + entropy).
pub const TRACE_ID_TOTAL_LENGTH: usize = 32;
/// Crockford base32 alphabet, lowercase (no i, l, o, u).
const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
/// A unique-per-request trace identifier, eg. `trace_1h3x9k2m4p7q8r5t6v0w2y3z4a`.
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub struct TraceId(String);
impl TraceId {
/// Generate a new random trace id. Infallible.
#[inline]
pub fn generate() -> Self {
use rand::Rng;
let mut rng = rand::thread_rng();
let entropy: String = (0..TRACE_ID_ENTROPY_CHARS)
.map(|_| {
let idx = rng.gen_range(0..CROCKFORD_LOWERCASE_CHARSET.len());
CROCKFORD_LOWERCASE_CHARSET[idx] as char
})
.collect();
TraceId(format!("{}{}", TRACE_ID_PREFIX, entropy))
}
#[inline]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for TraceId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0)
}
}
impl fmt::Debug for TraceId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0)
}
}
tokio::task_local! {
/// The trace id for the currently-executing request, if any.
///
/// Set by the HTTP middleware via [`TRACE_ID_TASK_LOCAL.scope(...)`], and
/// readable from anywhere within the request's task — including the
/// logger's format closure and the pager — via [`current_trace_id`].
pub static TRACE_ID_TASK_LOCAL: TraceId;
}
/// The trace id of the currently-executing request, if one is in scope.
///
/// Returns `None` outside of a request context (startup, background threads,
/// detached `tokio::spawn` tasks that weren't explicitly re-scoped).
#[inline]
pub fn current_trace_id() -> Option<TraceId> {
TRACE_ID_TASK_LOCAL.try_with(|trace_id| trace_id.clone()).ok()
}
#[cfg(test)]
mod tests {
use super::*;
mod generation {
use super::*;
#[test]
fn total_length_is_32() {
assert_eq!(TraceId::generate().as_str().len(), TRACE_ID_TOTAL_LENGTH);
}
#[test]
fn has_prefix() {
assert!(TraceId::generate().as_str().starts_with(TRACE_ID_PREFIX));
}
#[test]
fn entropy_uses_crockford_lowercase_only() {
let trace_id = TraceId::generate();
let entropy = &trace_id.as_str()[TRACE_ID_PREFIX.len()..];
assert_eq!(entropy.len(), TRACE_ID_ENTROPY_CHARS);
for c in entropy.chars() {
assert!(
CROCKFORD_LOWERCASE_CHARSET.contains(&(c as u8)),
"unexpected character: {}",
c
);
}
}
#[test]
fn ids_are_unique() {
let a = TraceId::generate();
let b = TraceId::generate();
assert_ne!(a, b);
}
}
mod task_local {
use super::*;
#[tokio::test]
async fn absent_outside_scope() {
assert!(current_trace_id().is_none());
}
#[tokio::test]
async fn present_inside_scope() {
let trace_id = TraceId::generate();
let expected = trace_id.clone();
TRACE_ID_TASK_LOCAL
.scope(trace_id, async move {
assert_eq!(current_trace_id(), Some(expected));
})
.await;
}
#[tokio::test]
async fn survives_await_points() {
let trace_id = TraceId::generate();
let expected = trace_id.clone();
TRACE_ID_TASK_LOCAL
.scope(trace_id, async move {
tokio::task::yield_now().await;
assert_eq!(current_trace_id(), Some(expected));
})
.await;
}
}
}
@@ -89,6 +89,7 @@ seedance2pro_client.workspace = true
server_environment = { path = "../../../lib/server_environment" }
thumbnail_generator = { path = "../../../api_clients/thumbnail_generator" }
tokens.workspace = true
trace_id.workspace = true
tts_common = { path = "../../../lib/deprecated/tts_common" }
url_config = { path = "../../../lib/url_config" }
url_utils = { path = "../../../lib/url_utils" }
@@ -16,7 +16,6 @@ use crate::http_server::common_responses::tag_info::TagInfo;
use crate::http_server::common_responses::user_details_lite::{UserDefaultAvatarInfo, UserDetailsLight};
use crate::http_server::deprecated_endpoints::conversion::enqueue_fbx_to_gltf_handler::*;
use crate::http_server::deprecated_endpoints::engine::create_scene_handler::*;
use crate::http_server::deprecated_endpoints::image_gen::enqueue_image_generation::EnqueueImageGenRequestSuccessResponse;
use crate::http_server::deprecated_endpoints::workflows::enqueue::enqueue_face_fusion_workflow_handler::*;
use crate::http_server::deprecated_endpoints::workflows::enqueue::enqueue_live_portrait_workflow_handler::*;
use crate::http_server::deprecated_endpoints::workflows::enqueue::vst_common::vst_error::*;
@@ -88,7 +87,10 @@ use crate::http_server::endpoints::media_files::upsert_upload::write_error::Medi
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*;
use crate::http_server::endpoints::model_download::enqueue_gptsovits_model_download_handler::*;
use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::*;
use crate::http_server::endpoints::moderation::debug_logs::moderation_list_debug_logs_for_token_handler::*;
use artcraft_api_defs::moderation::debug_logs::debug_log_entry::*;
use artcraft_api_defs::moderation::debug_logs::moderation_list_all_debug_logs::*;
use artcraft_api_defs::moderation::debug_logs::moderation_list_debug_logs_for_token::*;
use artcraft_api_defs::moderation::debug_logs::moderation_list_debug_logs_for_user::*;
use crate::http_server::endpoints::moderation::jobs::moderation_get_job_by_token_handler::*;
use crate::http_server::endpoints::moderation::user_feature_flags::moderator_edit_user_feature_flags_handler::*;
use crate::http_server::endpoints::moderation::user_feature_flags::moderator_list_all_available_user_feature_flags_handler::*;
@@ -134,6 +136,7 @@ use artcraft_api_defs::moderation::user_spend_summaries::get_summary::*;
use artcraft_api_defs::moderation::user_spend_summaries::reengagement_list::*;
use artcraft_api_defs::moderation::user_spend_summaries::top_users_list::*;
use artcraft_api_defs::moderation::user_daily_spends::user_daily_spends_list::*;
use artcraft_api_defs::moderation::top_spenders::list::*;
use artcraft_api_defs::moderation::user_spend_events::list::*;
use artcraft_api_defs::folders::subfolder::*;
use artcraft_api_defs::moderation::user_referrals::list_global_user_referrals::*;
@@ -214,8 +217,6 @@ use billing_component::stripe::http_endpoints::checkout::create::stripe_create_c
use crate::http_server::endpoints::billing_fakeyou::list_active_user_subscriptions_handler::*;
use crate::http_server::endpoints::service::status_alert_handler::*;
use crate::http_server::endpoints::stats::get_unified_queue_stats_handler::*;
use crate::http_server::endpoints::studio_gen2::enqueue_studio_gen2_handler::*;
use crate::http_server::endpoints::studio_gen2::enqueue_studio_gen2_handler::EnqueueStudioGen2Request;
use crate::http_server::endpoints::tags::list_tags_for_entity_handler::*;
use crate::http_server::endpoints::tags::set_tags_for_entity_handler::*;
use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::*;
@@ -238,9 +239,6 @@ use crate::http_server::endpoints::users::login_handler::LoginErrorResponse;
use crate::http_server::endpoints::users::logout_handler::*;
use crate::http_server::endpoints::users::session_info_handler::*;
use crate::http_server::endpoints::users::session_token_info_handler::*;
use crate::http_server::endpoints::voice_conversion::enqueue_voice_conversion_inference_handler::*;
use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*;
use crate::http_server::endpoints::voice_designer::voice_datasets::list_datasets_by_user::*;
use crate::http_server::endpoints::weights::delete::delete_weight_handler::*;
use crate::http_server::endpoints::weights::get::get_weight_handler::*;
use crate::http_server::endpoints::weights::list::list_available_weights_handler::*;
@@ -385,7 +383,6 @@ use crate::http_server::endpoints::video_info::video_info_upload_handler::*;
use crate::http_server::endpoints::video_info::video_info_notes_handler::*;
use crate::http_server::endpoints::web_referrals::log_web_referral_handler::*;
use crate::http_server::endpoints::image_studio::update_gpt_image_job_status_handler::*;
use crate::http_server::endpoints::voice_conversion::enqueue_seed_vc_inference_handler::*;
use crate::http_server::endpoints::credits::get_session_credits_handler::*;
use crate::http_server::endpoints::subscriptions::get_session_subscription_handler::*;
use crate::http_server::endpoints::analytics::log_app_active_user_handler::*;
@@ -399,6 +396,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::moderation::user_spend_summaries::moderator_reengagement_list_handler::moderator_reengagement_list_handler,
crate::http_server::endpoints::moderation::user_spend_summaries::moderator_top_users_list_handler::moderator_top_users_list_handler,
crate::http_server::endpoints::moderation::user_daily_spends::moderator_user_daily_spends_handler::moderator_user_daily_spends_handler,
crate::http_server::endpoints::moderation::top_spenders::moderator_list_top_spenders_handler::moderator_list_top_spenders_handler,
crate::http_server::endpoints::moderation::user_spend_events::moderator_list_user_spend_events_handler::moderator_list_user_spend_events_handler,
billing_component::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_json_handler::stripe_create_checkout_session_json_handler,
crate::http_server::endpoints::billing_fakeyou::list_active_user_subscriptions_handler::list_active_user_subscriptions_handler,
@@ -484,7 +482,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::prompts::get_prompt_handler::get_prompt_handler,
crate::http_server::endpoints::service::status_alert_handler::status_alert_handler,
crate::http_server::endpoints::stats::get_unified_queue_stats_handler::get_unified_queue_stats_handler,
crate::http_server::endpoints::studio_gen2::enqueue_studio_gen2_handler::enqueue_studio_gen2_handler,
crate::http_server::endpoints::tags::list_tags_for_entity_handler::list_tags_for_entity_handler,
crate::http_server::endpoints::tags::set_tags_for_entity_handler::set_tags_for_entity_handler,
crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::enqueue_infer_tts_handler,
@@ -506,9 +503,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::users::logout_handler::logout_handler,
crate::http_server::endpoints::users::session_info_handler::session_info_handler,
crate::http_server::endpoints::users::session_token_info_handler::session_token_info_handler,
crate::http_server::endpoints::voice_conversion::enqueue_voice_conversion_inference_handler::enqueue_voice_conversion_inference_handler,
crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request,
crate::http_server::endpoints::voice_designer::voice_datasets::list_datasets_by_user::list_datasets_by_user_handler,
crate::http_server::endpoints::weights::delete::delete_weight_handler::delete_weight_handler,
crate::http_server::endpoints::weights::get::get_weight_handler::get_weight_handler,
crate::http_server::endpoints::weights::list::list_available_weights_handler::list_available_weights_handler,
@@ -564,9 +558,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::analytics::log_app_active_user_handler::log_app_active_user_handler,
crate::http_server::endpoints::analytics::log_app_active_user_json_handler::log_app_active_user_json_handler,
// Voice Conversion
crate::http_server::endpoints::voice_conversion::enqueue_seed_vc_inference_handler::enqueue_infer_seed_vc_handler,
// TTS
crate::http_server::endpoints::tts::enqueue_infer_f5_tts_handler::enqueue_infer_f5_tts_handler::enqueue_infer_f5_tts_handler,
// Characters
crate::http_server::endpoints::characters::list_characters_handler::list_characters_handler,
crate::http_server::endpoints::characters::create_character_handler::create_character_handler,
@@ -602,7 +594,9 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::moderation::wallets::moderator_add_banked_balance_to_wallet_handler::moderator_add_banked_balance_to_wallet_handler,
crate::http_server::endpoints::moderation::wallets::moderator_create_wallet_for_user_handler::moderator_create_wallet_for_user_handler,
crate::http_server::endpoints::moderation::wallets::moderator_get_wallet_handler::moderator_get_wallet_handler,
crate::http_server::endpoints::moderation::debug_logs::moderation_list_all_debug_logs_handler::moderation_list_all_debug_logs_handler,
crate::http_server::endpoints::moderation::debug_logs::moderation_list_debug_logs_for_token_handler::moderation_list_debug_logs_for_token_handler,
crate::http_server::endpoints::moderation::debug_logs::moderation_list_debug_logs_for_user_handler::moderation_list_debug_logs_for_user_handler,
crate::http_server::endpoints::moderation::jobs::moderation_get_job_by_token_handler::moderation_get_job_by_token_handler,
crate::http_server::endpoints::moderation::staff_audit_logs::moderator_list_staff_audit_logs_handler::moderator_list_staff_audit_logs_handler,
crate::http_server::endpoints::moderation::user_referrals::moderator_list_global_user_referrals_handler::moderator_list_global_user_referrals_handler,
@@ -663,6 +657,9 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
TopUserEntry,
TopUsersWindow,
ModeratorUserDailySpendsResponse,
ModeratorListTopSpendersResponse,
TopSpenderEntry,
TopSpendersWindow,
ModeratorListUserSpendEventsResponse,
UserSpendEventListEntry,
UserDailySpendEntry,
@@ -922,21 +919,13 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
EnqueueFbxToGltfRequestSuccessResponse,
EnqueueGptSovitsModelDownloadRequest,
EnqueueGptSovitsModelDownloadSuccessResponse,
EnqueueImageGenRequestSuccessResponse,
EnqueueLivePortraitCropDimensions,
EnqueueLivePortraitWorkflowRequest,
EnqueueLivePortraitWorkflowSuccessResponse,
EnqueueStudioGen2Request,
EnqueueStudioGen2Response,
EnqueueTTSRequest,
EnqueueTTSRequestSuccessResponse,
EnqueueVideoStyleTransferRequest,
EnqueueVideoStyleTransferSuccessResponse,
EnqueueVoiceConversionInferenceRequest,
EnqueueVoiceConversionInferenceSuccessResponse,
FeaturedMediaFile,
FeaturedModelWeightForList,
FundamentalFrequencyMethod,
GenerateFlux1DevTextToImageAspectRatio,
GenerateFlux1DevTextToImageNumImages,
GenerateFlux1DevTextToImageRequest,
@@ -994,15 +983,12 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
GetProfilePathInfo,
GetPromptPathInfo,
GetPromptSuccessResponse,
GetUnifiedQueueStatsError,
GetUnifiedQueueStatsSuccessResponse,
GetUserRatingResponse,
GetWeightPathInfo,
GetWeightResponse,
GoogleCreateAccountRequest,
GoogleCreateAccountSuccessResponse,
InferTtsError,
InferTtsRequest,
InferTtsSuccessResponse,
InferenceJobStatusResponsePayload,
InferenceJobTokenType,
@@ -1016,9 +1002,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
ListCommentsError,
ListCommentsPathInfo,
ListCommentsSuccessResponse,
ListDatasetsByUserError,
ListDatasetsByUserPathInfo,
ListDatasetsByUserSuccessResponse,
ListFeaturedMediaFilesQueryParams,
ListFeaturedMediaFilesSuccessResponse,
ListFeaturedWeightsError,
@@ -1030,10 +1013,17 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
GetJobByTokenPathInfo,
GetJobByTokenSuccessResponse,
ModerationJobResponse,
ListDebugLogsPathInfo,
ListDebugLogsQueryParams,
ListDebugLogsSuccessResponse,
DebugLogEntry,
ModerationDebugLogEntry,
ModerationDebugLogUser,
ModerationListAllDebugLogsEntry,
ModerationListAllDebugLogsQueryParams,
ModerationListAllDebugLogsSuccessResponse,
ModerationListDebugLogsForTokenPathInfo,
ModerationListDebugLogsForTokenQueryParams,
ModerationListDebugLogsForTokenSuccessResponse,
ModerationListDebugLogsForUserPathInfo,
ModerationListDebugLogsForUserQueryParams,
ModerationListDebugLogsForUserSuccessResponse,
ListStaffAuditLogsQueryParams,
ListStaffAuditLogsSuccessResponse,
ListImpersonationRequestsQueryParams,
@@ -1182,7 +1172,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
WeightsData,
WriteEngineAssetMediaSuccessResponse,
WriteSceneFileMediaSuccessResponse,
ZsDatasetRecord,
// Cost Estimate types
EstimateImageCostRequest,
@@ -1,143 +0,0 @@
// NB: Incrementally getting rid of build warnings...
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
use std::sync::Arc;
use actix_web::web::Path;
use actix_web::{web, HttpRequest, HttpResponse};
use chrono::{DateTime, Utc};
use log::warn;
use mysql_queries::queries::tts::tts_category_assignments::list_assigned_tts_categories_query_builder::ListAssignedTtsCategoriesQueryBuilder;
use crate::state::server_state::ServerState;
use crate::http_server::common_responses::common_web_error::CommonWebError;
// =============== Request ===============
/// For the URL PathInfo
#[derive(Deserialize)]
pub struct ListTtsModelAssignedCategoriesPathInfo {
/// TTS model token
token: String,
}
// =============== Success Response ===============
#[derive(Serialize)]
pub struct ListTtsModelAssignedCategoriesResponse {
pub success: bool,
pub categories: Vec<DisplayCategory>,
}
/// Public-facing and optimized (non-irrelevant fields) category list
/// Used for the main TTS dropdown as well as the TTS edit/tagging UI
#[derive(Serialize)]
pub struct DisplayCategory {
pub category_token: String,
pub model_type: String, // TODO: ENUM
pub maybe_super_category_token: Option<String>,
pub can_directly_have_models: bool,
pub can_have_subcategories: bool,
pub can_only_mods_apply: bool,
pub name: String,
pub maybe_dropdown_name: Option<String>,
// NB: It's okay for non-mods to see this.
pub is_mod_approved: Option<bool>,
pub category_created_at: DateTime<Utc>,
pub category_updated_at: DateTime<Utc>,
pub category_deleted_at: Option<DateTime<Utc>>,
//pub creator_user_token: Option<String>,
//pub creator_username: Option<String>,
//pub creator_display_name: Option<String>,
}
// =============== Error Response ===============
// NB: Not using derive_more::Display since Clion doesn't understand it.
// =============== Handler ===============
pub async fn list_tts_model_assigned_categories_handler(
http_request: HttpRequest,
path: Path<ListTtsModelAssignedCategoriesPathInfo>,
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, CommonWebError>
{
let mut mysql_connection = server_state.mysql_pool.acquire()
.await
.map_err(|e| {
warn!("Could not acquire DB pool: {:?}", e);
CommonWebError::from_error(e)
})?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::from_error(e)
})?;
// TODO: We don't have any permissions for categories. This is a proxy.
let is_mod = maybe_user_session
.map(|session| session.can_ban_users)
.unwrap_or(false);
let query_builder = ListAssignedTtsCategoriesQueryBuilder::for_model_token(&path.token)
.show_invalid_model_not_allowed_categories(is_mod)
.show_deleted(is_mod)
.show_unapproved(is_mod);
let query_result =
query_builder.perform_query_by_connection(&mut mysql_connection).await;
let results = match query_result {
Ok(results) => results,
Err(err) => {
warn!("DB error: {:?}", err);
return Err(CommonWebError::from_anyhow_error(err));
}
};
let mut categories = results.categories.into_iter()
.map(|c| {
DisplayCategory {
category_token: c.category_token,
model_type: c.model_type,
maybe_super_category_token: c.maybe_super_category_token,
can_directly_have_models: c.can_directly_have_models,
can_have_subcategories: c.can_have_subcategories,
can_only_mods_apply: c.can_only_mods_apply,
name: c.name.clone(),
maybe_dropdown_name:c.maybe_dropdown_name,
is_mod_approved: c.is_mod_approved,
category_created_at: c.category_created_at,
category_updated_at: c.category_updated_at,
category_deleted_at: c.category_deleted_at,
}
})
.collect::<Vec<DisplayCategory>>();
// TODO: Sort by dropdown name too.
categories.sort_by(|c1, c2| c1.name.cmp(&c2.name));
let response = ListTtsModelAssignedCategoriesResponse {
success: true,
categories,
};
let body = serde_json::to_string(&response)
.map_err(CommonWebError::from_error)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body))
}
@@ -1,4 +1,3 @@
pub mod assign_tts_category;
pub mod list_fully_computed_assigned_tts_categories;
pub mod list_tts_categories;
pub mod list_tts_model_assigned_categories;
@@ -1,75 +0,0 @@
use std::collections::BTreeSet;
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Debug, Deserialize, Serialize, ToSchema)]
pub enum EnqueueImageGenType {
#[serde(rename = "upload_lora")]
UploadLoRA,
#[serde(rename = "upload_sd")]
Checkpoint,
#[serde(rename = "inference")]
Inference,
}
impl EnqueueImageGenType {
pub fn to_str(&self) -> &'static str {
match self {
Self::UploadLoRA => "upload_lora",
Self::Checkpoint => "upload_sd",
Self::Inference => "inference",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"upload_lora" => Ok(Self::UploadLoRA),
"upload_sd" => Ok(Self::Checkpoint),
"inference" => Ok(Self::Inference),
_ => Err(format!("invalid value: {:?}", value)),
}
}
pub fn all_variants() -> BTreeSet<Self> {
BTreeSet::from([
Self::UploadLoRA,
Self::Checkpoint,
Self::Inference
])
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_to_str() {
assert_eq!(EnqueueImageGenType::UploadLoRA.to_str(), "upload_lora");
assert_eq!(EnqueueImageGenType::Checkpoint.to_str(), "upload_sd");
assert_eq!(EnqueueImageGenType::Inference.to_str(), "inference");
}
#[test]
fn test_from_str() {
assert_eq!(EnqueueImageGenType::from_str("upload_lora").unwrap(), EnqueueImageGenType::UploadLoRA);
assert_eq!(EnqueueImageGenType::from_str("upload_sd").unwrap(), EnqueueImageGenType::Checkpoint);
assert_eq!(EnqueueImageGenType::from_str("inference").unwrap(), EnqueueImageGenType::Inference);
assert!(EnqueueImageGenType::from_str("invalid").is_err());
}
#[test]
fn test_all_variants() {
let variants = EnqueueImageGenType::all_variants();
assert_eq!(variants.len(), 3);
assert!(variants.contains(&EnqueueImageGenType::UploadLoRA));
assert!(variants.contains(&EnqueueImageGenType::Checkpoint));
assert!(variants.contains(&EnqueueImageGenType::Inference));
}
}
@@ -1,560 +0,0 @@
#![forbid(unused_mut)]
use std::fmt::Debug;
use std::fmt::{Display, Formatter};
use std::sync::Arc;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Json;
use actix_web::{web, HttpRequest, HttpResponse};
use log::error;
use log::warn;
use serde::Deserialize;
use serde::Serialize;
use utoipa::ToSchema;
use web::Data;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use http_server_common::request::get_request_ip::get_request_ip;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::{
GenericInferenceArgs,
InferenceCategoryAbbreviated,
PolymorphicInferenceArgs,
};
use mysql_queries::payloads::generic_inference_args::inner_payloads::image_generation_payload::StableDiffusionArgs;
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{
insert_generic_inference_job,
InsertGenericInferenceArgs,
};
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::state::server_state::ServerState;
/// This is the number of images (batch size) to generate for each request.
/// We should allow all users to have multiple images generated at once as this
/// is what other providers do.
const MINIMUM_IMAGE_COUNT : u32 = 4;
/// The maximum number of images (batch size) to generate for each request.
const MAXIMUM_IMAGE_COUNT : u32 = 8;
/// Debug requests can get routed to special "debug-only" workers, which can
/// be used to trial new code, run debugging, etc.
const DEBUG_HEADER_NAME: &str = "enable-debug-mode";
/// The routing tag header can send workloads to particular k8s hosts.
/// This is useful for catching the live logs or intercepting the job.
const ROUTING_TAG_HEADER_NAME: &str = "routing-tag";
#[derive(Debug, Deserialize, Clone, Copy, Eq, PartialEq)]
pub enum TypeOfInference {
#[serde(rename = "lora")]
Lora,
#[serde(rename = "model")]
Model,
#[serde(rename = "inference")]
Inference
}
impl Display for TypeOfInference {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
match self {
TypeOfInference::Lora => write!(f, "lora"),
TypeOfInference::Model => write!(f, "model"),
TypeOfInference::Inference => write!(f, "inference"),
}
}
}
pub fn is_valid_string(input: &str) -> bool {
let valid_strings = [
"DPM++ 2M Karras",
"DPM++ SDE Karras",
"DPM++ 2M SDE Exponential",
"DPM++ 2M SDE Karras",
"Euler a",
"Euler",
"LMS",
"Heun",
"DPM2",
"DPM2 a",
"DPM++ 2S a",
"DPM++ 2M",
"DPM++ SDE",
"DPM++ 2M SDE",
"DPM++ 2M SDE Heun",
"DPM++ 2M SDE Heun Karras",
"DPM++ 2M SDE Heun Exponential",
"DPM++ 3M SDE",
"DPM++ 3M SDE Karras",
"DPM++ 3M SDE Exponential",
"DPM fast",
"DPM adaptive",
"LMS Karras",
"DPM2 Karras",
"DPM2 a Karras",
"DPM++ 2S a Karras",
];
valid_strings.contains(&input)
}
#[derive(Deserialize, Default, ToSchema)]
pub struct EnqueueImageGenRequest {
pub uuid_idempotency_token: String,
pub maybe_image_source: Option<String>,
pub maybe_sd_model_token: Option<String>,
pub maybe_lora_model_token: Option<String>,
pub maybe_prompt: Option<String>,
pub maybe_n_prompt: Option<String>,
pub maybe_seed: Option<i64>,
pub maybe_width: Option<u32>,
pub maybe_height: Option<u32>,
pub maybe_sampler: Option<String>,
pub maybe_upload_path: Option<String>,
pub maybe_lora_upload_path: Option<String>,
pub maybe_cfg_scale: Option<u32>,
pub maybe_number_of_samples: Option<u32>,
pub maybe_batch_count: Option<u32>,
pub maybe_name: Option<String>,
pub maybe_description: Option<String>,
pub maybe_version: Option<u32>,
// Optional cover image on LoRA or SD upload
pub maybe_cover_image_media_file_token: Option<MediaFileToken>,
}
#[derive(Serialize, ToSchema)]
pub struct EnqueueImageGenRequestSuccessResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
// Implementation for enqueuing a TTS request
// Reference enqueue_infer_tts_handler.rs for checks: rate limiting / user sessions
// insert generic inference job.rs
// Need to convert it to generic inference job.
#[utoipa::path(
post,
path = "/inference/enqueue_image_gen/",
responses(
(
status = 200,
description = "Enqueue TTS generically",
body = EnqueueImageGenRequestSuccessResponse,
),
(status = 400, description = "Bad input", body = CommonWebError),
(status = 401, description = "Not authorized", body = CommonWebError),
(status = 429, description = "Rate limited", body = CommonWebError),
(status = 500, description = "Server error", body = CommonWebError)
),
params(("request" = EnqueueImageGenRequest, description = "Payload for TTS Request"))
)]
pub async fn enqueue_image_generation_request(
http_request: HttpRequest,
request: Json<EnqueueImageGenRequest>,
server_state: Data<Arc<ServerState>>
) -> Result<HttpResponse, CommonWebError> {
let inference_mode = inference_mode_from_http_request(&http_request)
.ok_or(CommonWebError::BadInputWithSimpleMessage("Invalid request".to_string()))?;
validate_request(&request, inference_mode)?;
// TODO: Brandon need to figure out premium vs not premium
let mut maybe_user_token: Option<UserToken> = None;
let visbility = enums::common::visibility::Visibility::Public;
let mut mysql_connection = server_state.mysql_pool.acquire().await.map_err(|err| {
warn!("MySql pool error: {:?}", err);
CommonWebError::from_error(err)
})?;
// ==================== USER SESSION ==================== //
let maybe_user_session = server_state.session_checker
.maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection).await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::from_error(e)
})?;
if let Some(user_session) = maybe_user_session.as_ref() {
maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token));
}
// ==================== PLANS ==================== //
// Plan should handle "first anonymous use" and "investor" cases.
let plan = get_correct_plan_for_session(
server_state.server_environment,
maybe_user_session.as_ref()
);
// Separate priority for animation.
let priority_level = plan.web_vc_base_priority_level();
// ==================== DEBUG MODE + ROUTING TAG ==================== //
let is_debug_request = get_request_header_optional(&http_request, DEBUG_HEADER_NAME).is_some();
let maybe_routing_tag = get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME).map(
|routing_tag| routing_tag.trim().to_string()
);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.role.is_banned {
return Err(CommonWebError::NotAuthorized);
}
}
// DETECT premium user and queue
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.logged_out,
Some(ref _user) => &server_state.redis_rate_limiters.logged_in,
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request).await {
return Err(CommonWebError::TooManyRequests);
}
// Get up IP address
let ip_address = get_request_ip(&http_request);
// Check the inference args to make sure everything is all there for upload loRA / model or standard inference
// ==================== HANDLE IDEMPOTENCY ==================== //
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string())
})?;
// ==================== INFERENCE ARGS ==================== //
let width = request.maybe_width.unwrap_or(512);
let height = request.maybe_height.unwrap_or(512);
let width = match width {
0..=512 => 512,
513..=768 => 768,
769..=1024 => 1024,
_ => 1024,
};
let height = match height {
0..=512 => 512,
513..=768 => 768,
769..=1024 => 1024,
_ => 1024,
};
let cfg_scale = match request.maybe_cfg_scale {
Some(val) => if val > 32 { 32 } else { val }
None => 7,
};
let number_of_samples = match request.maybe_number_of_samples {
Some(val) => if val > 128 { 128 } else { val }
None => 20,
};
let batch_count = match request.maybe_batch_count {
None => MINIMUM_IMAGE_COUNT, // NB: Default to "3" images for everyone
Some(0...MINIMUM_IMAGE_COUNT) => MINIMUM_IMAGE_COUNT,
Some(val) => if val > MAXIMUM_IMAGE_COUNT { MAXIMUM_IMAGE_COUNT } else { val }
};
let sampler = match request.maybe_sampler.clone() {
Some(val) => if is_valid_string(&val) { val.clone() } else { String::from("Euler a") }
None => String::from("Euler a"),
};
let sd_weight_token = ModelWeightToken(
request.maybe_sd_model_token.clone().unwrap_or_default()
);
let lora_token = ModelWeightToken(request.maybe_lora_model_token.clone().unwrap_or_default());
let n_prompt = request.maybe_n_prompt.clone().unwrap_or_default();
// we can only do 1 upload type at a time.
// if we are uploading a model.
let mut maybe_sd_upload_url = None;
let mut maybe_lora_upload_url = None;
let both_fields = (
request.maybe_upload_path.as_deref(),
request.maybe_lora_upload_path.as_deref()
);
match both_fields {
(Some(_), Some(_)) => {
return Err(CommonWebError::BadInputWithSimpleMessage("Can't upload both lora and model".to_string()));
}
(Some(sd_url), None) => {
maybe_sd_upload_url = Some(sd_url.to_string());
}
(None, Some(lora_url)) => {
maybe_lora_upload_url = Some(lora_url.to_string());
}
_ => {}, // No-op for inference
}
// NB: This is done to populate the new top-level field in the jobs table.
// We won't read this yet, but in the meantime it can serve as analytics, and in the future
// we can switch to it.
let maybe_either_download_url = maybe_sd_upload_url.as_deref()
.or(maybe_lora_upload_url.as_deref())
.map(|s| s.to_string());
let seed = request.maybe_seed.unwrap_or(-1);
let mut version : u32 = 0;
let type_of_inference = match inference_mode {
TypeOfInference::Lora => "lora",
TypeOfInference::Model => "model",
TypeOfInference::Inference => "inference",
};
if let Some(s) = request.maybe_version {
version = s;
}
let inference_args = StableDiffusionArgs {
maybe_sd_model_token: Some(sd_weight_token),
maybe_lora_model_token: Some(lora_token),
maybe_prompt: request.maybe_prompt.clone(),
maybe_n_prompt: request.maybe_n_prompt.clone(),
maybe_seed: Some(seed),
maybe_upload_path: maybe_sd_upload_url,
maybe_lora_upload_path: maybe_lora_upload_url,
type_of_inference: type_of_inference.to_string(),
maybe_cfg_scale: Some(cfg_scale),
maybe_number_of_samples: Some(number_of_samples),
maybe_batch_count: Some(batch_count),
maybe_width: Some(width),
maybe_height: Some(height),
maybe_sampler: Some(sampler),
maybe_description: request.maybe_description.clone(),
maybe_name: request.maybe_name.clone(),
maybe_version: Some(version),
};
// create the inference args here
let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request);
// create the job record here!
let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
job_type: InferenceJobType::StableDiffusion,
maybe_product_category: None, // This is not a product anymore
inference_category: InferenceCategory::ImageGeneration,
maybe_model_type: Some(InferenceModelType::StableDiffusion), // NB: Model is static during inference
maybe_model_token: None, // NB: Model is static during inference
maybe_input_source_token: None,
maybe_input_source_token_type: None,
maybe_download_url: maybe_either_download_url.as_deref(),
maybe_cover_image_media_file_token: request.maybe_cover_image_media_file_token.as_ref(),
maybe_raw_inference_text: None,
maybe_max_duration_seconds: None,
maybe_inference_args: Some(GenericInferenceArgs {
inference_category: Some(InferenceCategoryAbbreviated::ImageGeneration),
args: Some(PolymorphicInferenceArgs::Ig(inference_args)),
}),
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: visbility,
priority_level,
requires_keepalive: false, //reverse ... TODO fix this. we set it base on account is premium or not ...
is_debug_request,
maybe_routing_tag: maybe_routing_tag.as_deref(),
mysql_pool: &server_state.mysql_pool,
}).await;
let job_token = match query_result {
Ok((job_token, _id)) => job_token,
Err(err) => {
warn!("New generic inference job creation DB error: {:?}", err);
if err.had_duplicate_idempotency_token() {
return Err(CommonWebError::BadInputWithSimpleMessage("Duplicate idempotency token".to_string()));
}
return Err(CommonWebError::from_error(err));
}
};
let response: EnqueueImageGenRequestSuccessResponse = EnqueueImageGenRequestSuccessResponse {
success: true,
inference_job_token: job_token,
};
let body = serde_json
::to_string(&response)
?;
// Error handling 101 rust result type returned like so.
Ok(HttpResponse::Ok().content_type("application/json").body(body))
}
fn validate_request(
request: &Json<EnqueueImageGenRequest>,
inference_mode: TypeOfInference
) -> Result<(), CommonWebError> {
let mut requires_name = false;
let mut requires_description = false;
let mut requires_sd_model_token = false;
let mut requires_lora_upload_path = false;
let mut requires_upload_path = false;
match inference_mode {
TypeOfInference::Lora => {
requires_name = true;
requires_description = true;
requires_lora_upload_path = true;
}
TypeOfInference::Model => {
requires_name = true;
requires_description = true;
requires_upload_path = true;
}
TypeOfInference::Inference => {
requires_sd_model_token = true;
}
}
if requires_name && request.maybe_name.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage("Missing Model / Lora Name".to_string()));
}
if requires_description && request.maybe_description.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage("Missing Model / Lora Description".to_string()));
}
if requires_sd_model_token && request.maybe_sd_model_token.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage("Missing Model Token".to_string()));
}
if requires_lora_upload_path && request.maybe_lora_upload_path.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage("Missing Lora Upload Path".to_string()));
}
if requires_upload_path && request.maybe_upload_path.is_none() {
return Err(CommonWebError::BadInputWithSimpleMessage("Missing Model Upload Path".to_string()));
}
Ok(())
}
fn inference_mode_from_http_request(http_request: &HttpRequest) -> Option<TypeOfInference> {
inference_mode_from_url_path(http_request.path())
}
fn inference_mode_from_url_path(url_path: &str) -> Option<TypeOfInference> {
let last_segment = url_path.split("/").last();
match last_segment {
Some("lora") => Some(TypeOfInference::Lora),
Some("model") => Some(TypeOfInference::Model),
Some("inference") => Some(TypeOfInference::Inference),
_ => None
}
}
#[cfg(test)]
mod tests {
use crate::http_server::deprecated_endpoints::image_gen::enqueue_image_generation::{inference_mode_from_url_path, TypeOfInference};
#[test]
fn test_url_paths() {
// Valid routes
assert_eq!(inference_mode_from_url_path("/v1/image_gen/enqueue/inference"), Some(TypeOfInference::Inference));
assert_eq!(inference_mode_from_url_path("/v1/image_gen/upload/model"), Some(TypeOfInference::Model));
assert_eq!(inference_mode_from_url_path("/v1/image_gen/upload/lora"), Some(TypeOfInference::Lora));
// Non-routes
assert_eq!(inference_mode_from_url_path(""), None);
assert_eq!(inference_mode_from_url_path("/v1/image_gen/enqueue/foo"), None);
assert_eq!(inference_mode_from_url_path("/v1/image_gen/upload/foo"), None);
}
}
// with LoRA
// http://127.0.0.1:12345/v1/image_gen/inference/enqueue_image_gen
// {
// "uuid_idempotency_token": "12",
// "maybe_sd_model_token": "weight_dmmthavhawqc2hj7yqyemcbf8",
// "maybe_lora_model_token": "weight_t7gz78fjg27m0wtw6r33gafxs",
// "maybe_prompt": "raiden mei a very well drawn, anime girl, with pink and purple hair sitting down on a chair relaxed, highest quality, semi nude, masterpiece, painted",
// "maybe_a_prompt": "a anime girl, with pink and purple hair sitting down on a chair relaxed.",
// "maybe_n_prompt": "nsfw, black and white, low quality, pixelated",
// "maybe_seed": -1,
// "maybe_width": 1024,
// "maybe_height": 1024,
// "maybe_sampler": "DPM++ 2M SDE Heun",
// "maybe_cfg_scale": 7,
// "maybe_number_of_samples": 64,
// "maybe_batch_count": 4,
// }
// without LoRA
// {
// "uuid_idempotency_token": "132141",
// "maybe_sd_model_token": "weight_dmmthavhawqc2hj7yqyemcbf8",
// "maybe_prompt": "raiden mei a very well drawn, anime girl, with pink and purple hair sitting down on a chair relaxed, highest quality, semi nude, masterpiece, painted",
// "maybe_a_prompt": "a anime girl, with pink and purple hair sitting down on a chair relaxed.",
// "maybe_n_prompt": "nsfw, black and white, low quality, pixelated",
// "maybe_seed": -1,
// "maybe_width": 512,
// "maybe_height": 512,
// "maybe_sampler": "DPM++ 2M SDE Heun",
// "maybe_cfg_scale": 8,
// "maybe_number_of_samples": 64,
// "maybe_batch_count": 4
// }
// http://127.0.0.1:12345/v1/image_gen/upload/lora
// {
// "uuid_idempotency_token": "12",
// "maybe_lora_upload_path": "https://drive.google.com/file/d/1WRgR2pn0Ky8ls5_9Zq6tQlHTBvyWeach/view?usp=sharing",
// "maybe_name":"some_name",
// "maybe_description":"some_description"
// }
// http://127.0.0.1:12345/v1/image_gen/upload/model
// {
// "uuid_idempotency_token": "13",
// "maybe_upload_path": "https://drive.google.com/file/d/1WRgR2pn0Ky8ls5_9Zq6tQlHTBvyWeach/view?usp=sharing",
// "maybe_name":"some_name",
// "maybe_description":"some_description"
// }
// {
// "uuid_idempotency_token": "123",
// "maybe_lora_upload_path": "https://drive.google.com/file/d/1WRgR2pn0Ky8ls5_9Zq6tQlHTBvyWeach/view?usp=sharing",
// "maybe_name":"some_name",
// "maybe_description":"some_description"
// }
@@ -1,18 +0,0 @@
{
"uuid_idempotency_token": "123",
"type_of_inference": "checkpoint",
"maybe_sd_model_token": "weight_dmmthavhawqc2hj7yqyemcbf8",
"maybe_lora_model_token": "weight_t7gz78fjg27m0wtw6r33gafxs",
"maybe_prompt": "a very well drawn, anime girl, with pink and purple hair sitting down on a chair relaxed.",
"maybe_a_prompt": "a anime girl, with pink and purple hair sitting down on a chair relaxed.",
"maybe_n_prompt": "nsfw, black and white, low quality, pixelated",
"maybe_seed": "-1",
"maybe_width": "512",
"maybe_height": "512",
"maybe_sampler": "DPM++ 2M SDE Heun",
"maybe_upload_path": "https://drive.google.com/file/d/1WRgR2pn0Ky8ls5_9Zq6tQlHTBvyWeach/view?usp=sharing",
"maybe_lora_upload_path": "https://drive.google.com/file/d/1WRgR2pn0Ky8ls5_9Zq6tQlHTBvyWeach/view?usp=sharing",
"maybe_cfg_scale": "7",
"maybe_number_of_samples": "32",
"maybe_batch_count": "4"
}
@@ -1,2 +0,0 @@
pub mod enqueue_image_generation;
pub mod enqueue_generation_type;
@@ -1,18 +0,0 @@
pub const DEFAULT_INVESTOR_REDIRECT : &str = "https://fakeyou.com";
pub fn redirect_is_allowed(redirect_url: &str) -> bool {
match redirect_url {
"http://jungle.horse" => true,
"http://jungle.horse/" => true,
"https://create.storyteller.io" => true,
"https://create.storyteller.io/" => true,
"https://fakeyou.com" => true,
"https://fakeyou.com/clone" => true,
"https://jungle.horse" => true,
"https://jungle.horse/" => true,
"https://storyteller.io" => true,
"https://storyteller.io/" => true,
_ => false,
}
}

Some files were not shown because too many files have changed in this diff Show More