Compare commits

...
Author SHA1 Message Date
Michael Chung 30127e19e9 Fixed added weights type by request. 2023-12-23 23:25:43 -05:00
Michael Chung c50a2b36aa Add missing data in response
get weight endpoint to have updated_at (the list weight has but the single weight doesnt)
profile weight list endpoint weights_category is missing from the response, i need it for rendering the cards by what type of weight it is.
2023-12-23 14:46:45 -05:00
ArEnSc 49796c69e3 Merge pull request #239 from storytold/feature/weights_files
Feature/weights files
2023-12-21 20:51:39 -05:00
Brandon Thomas 268346f662 delete more easily 2023-12-21 19:40:33 -05:00
Brandon Thomas 7897ef730e refactor, fix tests 2023-12-21 19:04:56 -05:00
Brandon Thomas 43da5c91ad fix compile error 2023-12-21 18:47:28 -05:00
Brandon Thomas 71360879f3 cleanup 2023-12-21 18:43:18 -05:00
ArEnSc 7f80e106bd Merge pull request #238 from storytold/dec-21-mithoo
use public pagination for profile weights list endpoint
2023-12-21 14:30:17 -05:00
Kasisnu 58181b32cf use public pagination for profile weights list endpoint 2023-12-22 00:45:24 +05:30
ArEnSc 62c5d86584 Merge pull request #237 from storytold/dec-21-mithoo
Document user bookmarks endpoints
2023-12-21 12:34:48 -05:00
Brooks Palin cbb950cb40 Update general_setup.md
Fixed path for for storyteller-web
2023-12-21 10:30:36 -07:00
Kasisnu 64be9e49af Document user bookmarks endpoints 2023-12-21 19:52:44 +05:30
Brandon Thomas bd688703df Merge pull request #236 from storytold/brandon-creator-details
add creator details
2023-12-21 04:29:36 -05:00
Brandon Thomas c627c13e2d add creator details 2023-12-21 04:28:55 -05:00
Brandon Thomas 84104d91ac Merge pull request #235 from storytold/brandon-model-weight-avatars
model weight avatars
2023-12-21 04:13:58 -05:00
Brandon Thomas 64806135c5 model weight avatars 2023-12-21 04:13:44 -05:00
Brandon Thomas 5e48a6193a reorganize modules 2023-12-21 02:57:00 -05:00
Brandon Thomas 11c17f0a1d fix lints 2023-12-21 02:37:29 -05:00
Brandon Thomas cd91a25c3d Merge pull request #219 from storytold/dec-18-mithoo
document media files list endpoints, fix public page offset bug
2023-12-21 02:11:19 -05:00
Kasisnu c52d197717 sort user media file results by creation date 2023-12-21 12:35:44 +05:30
kasisnu 79d64c51ad fix bug with page offsets for public pagination 2023-12-21 12:35:44 +05:30
kasisnu 3e8d7be00e add remaining media files endpoints to openapispec 2023-12-21 12:35:42 +05:30
kasisnu 3a5b765e51 send favorite/comment counts with media file list endpoint 2023-12-21 12:33:04 +05:30
Brandon Thomas 6df4804e2a Merge pull request #228 from storytold/dec-19-mithoo
fail voice conversion jobs after model deleted
2023-12-21 02:02:17 -05:00
Brandon Thomas e6ac364622 Merge pull request #234 from storytold/brandon-rename-model-weights-bucket-fields
rename model_weights bucket fields: private -> public
2023-12-21 01:31:28 -05:00
Brandon Thomas 0cff684e97 rename model_weights bucket fields: private -> public 2023-12-21 01:30:10 -05:00
Brandon Thomas d698168f9e Merge pull request #233 from storytold/brandon-upload-media-file
media file upload
2023-12-21 00:40:02 -05:00
Brandon Thomas dd41c268e4 media file upload 2023-12-21 00:38:04 -05:00
Brandon Thomas d5280964bc Merge pull request #232 from storytold/brandon-fix-download-job-and-api
Fix download-job; fix Victor's API issue
2023-12-20 21:38:48 -05:00
Brandon Thomas c715d6795c Fix download-job; fix Victor's API issue 2023-12-20 21:37:48 -05:00
Brandon Thomas 4159191d68 Merge pull request #231 from storytold/brandon-tts-migration
work on tts model migration
2023-12-20 18:18:14 -05:00
Brandon Thomas 5e04cc6c2f work on tts model migration 2023-12-20 18:15:05 -05:00
Brandon Thomas c49ec5af2f Merge pull request #230 from storytold/brandon-run-rvc-svc-migration
migration cleanup
2023-12-19 23:23:20 -05:00
Brandon Thomas 44d75baeec update old record with token 2023-12-19 22:26:04 -05:00
Brandon Thomas 922e76eeb1 migration fix wip 2023-12-19 21:53:04 -05:00
Brandon Thomas 3b2f386cdb Merge pull request #229 from storytold/brandon-paths-fix
fix/cleanup some paths, fix tests
2023-12-19 20:44:35 -05:00
Brandon Thomas 61f30618b0 fix/cleanup some paths, fix tests 2023-12-19 20:43:27 -05:00
Kasisnu c116ad90d0 fail voice conversion jobs after model deleted 2023-12-19 16:45:38 +05:30
109 changed files with 3281 additions and 923 deletions
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.private_bucket_hash,\n wt.maybe_private_bucket_prefix,\n wt.maybe_private_bucket_extension,\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n WHERE\n wt.token = ?\n ",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.public_bucket_hash,\n wt.maybe_public_bucket_prefix,\n wt.maybe_public_bucket_extension,\n\n avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,\n avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,\n avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,\n\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n JOIN users\n ON users.token = wt.creator_user_token\n LEFT OUTER JOIN media_files as avatar\n ON avatar.token = wt.maybe_avatar_media_file_token\n WHERE\n wt.token = ?\n ",
"describe": {
"columns": [
{
@@ -85,6 +85,36 @@
},
{
"ordinal": 8,
"name": "creator_username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 9,
"name": "creator_display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 10,
"name": "creator_gravatar_hash",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 11,
"name": "creator_ip_address",
"type_info": {
"type": "VarString",
@@ -94,7 +124,7 @@
}
},
{
"ordinal": 9,
"ordinal": 12,
"name": "creator_set_visibility: enums::common::visibility::Visibility",
"type_info": {
"type": "String",
@@ -104,7 +134,7 @@
}
},
{
"ordinal": 10,
"ordinal": 13,
"name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
@@ -114,7 +144,7 @@
}
},
{
"ordinal": 11,
"ordinal": 14,
"name": "original_download_url",
"type_info": {
"type": "VarString",
@@ -124,7 +154,7 @@
}
},
{
"ordinal": 12,
"ordinal": 15,
"name": "original_filename",
"type_info": {
"type": "VarString",
@@ -134,7 +164,7 @@
}
},
{
"ordinal": 13,
"ordinal": 16,
"name": "file_size_bytes",
"type_info": {
"type": "Long",
@@ -144,7 +174,7 @@
}
},
{
"ordinal": 14,
"ordinal": 17,
"name": "file_checksum_sha2",
"type_info": {
"type": "String",
@@ -154,8 +184,8 @@
}
},
{
"ordinal": 15,
"name": "private_bucket_hash",
"ordinal": 18,
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
@@ -164,8 +194,8 @@
}
},
{
"ordinal": 16,
"name": "maybe_private_bucket_prefix",
"ordinal": 19,
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -174,8 +204,8 @@
}
},
{
"ordinal": 17,
"name": "maybe_private_bucket_extension",
"ordinal": 20,
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -184,7 +214,37 @@
}
},
{
"ordinal": 18,
"ordinal": 21,
"name": "maybe_avatar_public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 22,
"name": "maybe_avatar_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 23,
"name": "maybe_avatar_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 24,
"name": "cached_user_ratings_negative_count",
"type_info": {
"type": "Long",
@@ -194,7 +254,7 @@
}
},
{
"ordinal": 19,
"ordinal": 25,
"name": "cached_user_ratings_positive_count",
"type_info": {
"type": "Long",
@@ -204,7 +264,7 @@
}
},
{
"ordinal": 20,
"ordinal": 26,
"name": "cached_user_ratings_total_count",
"type_info": {
"type": "Long",
@@ -214,7 +274,7 @@
}
},
{
"ordinal": 21,
"ordinal": 27,
"name": "maybe_cached_user_ratings_ratio",
"type_info": {
"type": "Float",
@@ -224,7 +284,7 @@
}
},
{
"ordinal": 22,
"ordinal": 28,
"name": "cached_user_ratings_last_updated_at",
"type_info": {
"type": "Timestamp",
@@ -234,7 +294,7 @@
}
},
{
"ordinal": 23,
"ordinal": 29,
"name": "version",
"type_info": {
"type": "Long",
@@ -244,7 +304,7 @@
}
},
{
"ordinal": 24,
"ordinal": 30,
"name": "created_at",
"type_info": {
"type": "Timestamp",
@@ -254,7 +314,7 @@
}
},
{
"ordinal": 25,
"ordinal": 31,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
@@ -264,7 +324,7 @@
}
},
{
"ordinal": 26,
"ordinal": 32,
"name": "user_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -274,7 +334,7 @@
}
},
{
"ordinal": 27,
"ordinal": 33,
"name": "mod_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -298,6 +358,9 @@
false,
false,
false,
false,
false,
false,
true,
true,
true,
@@ -306,6 +369,9 @@
false,
true,
true,
true,
true,
true,
false,
false,
false,
@@ -318,5 +384,5 @@
true
]
},
"hash": "6a94791c8a8f66ee54bb367a4e8c71106bbcbbb205f54c14e3657279e83dabb6"
"hash": "4e30e34cdf7a5834a3981d5e634506966bac9f9c420e800d15ce313778272cee"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO model_weights\nSET\n token = ?,\n weights_type = ?,\n weights_category = ?,\n title = ?,\n maybe_thumbnail_token = NULL,\n maybe_avatar_media_file_token = NULL,\n maybe_cover_media_file_token = NULL,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n maybe_last_update_user_token = NULL,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n\n private_bucket_hash = ?,\n maybe_private_bucket_prefix = ?,\n maybe_private_bucket_extension = ?,\n cached_user_ratings_total_count = 0,\n cached_user_ratings_positive_count = 0,\n cached_user_ratings_negative_count = 0,\n maybe_cached_user_ratings_ratio = 0.0,\n cached_user_ratings_last_updated_at = NOW(),\n\n maybe_migration_old_model_token = ?,\n version = ?,\n created_at = ?,\n updated_at = ?,\n user_deleted_at = ?,\n mod_deleted_at = ?\n\nON DUPLICATE KEY UPDATE\n weights_type = ?,\n weights_category = ?,\n title = ?,\n maybe_thumbnail_token = NULL,\n maybe_avatar_media_file_token = NULL,\n maybe_cover_media_file_token = NULL,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n maybe_last_update_user_token = NULL,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n private_bucket_hash = ?,\n maybe_private_bucket_prefix = ?,\n maybe_private_bucket_extension = ?,\n cached_user_ratings_total_count = 0,\n cached_user_ratings_positive_count = 0,\n cached_user_ratings_negative_count = 0,\n maybe_cached_user_ratings_ratio = 0.0,\n cached_user_ratings_last_updated_at = NOW(),\n maybe_migration_old_model_token = ?,\n version = ?,\n created_at = ?,\n updated_at = ?,\n user_deleted_at = ?,\n mod_deleted_at = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 43
},
"nullable": []
},
"hash": "4f36ec7baa9fa98cca25b5cc11f09d0eaaa2dd5797374b4c1aa103999e8e1542"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n mw.title,\n mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n mw.maybe_thumbnail_token,\n mw.description_markdown,\n mw.description_rendered_html,\n users.token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_email_gravatar_hash,\n mw.creator_ip_address,\n mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n mw.original_download_url,\n mw.original_filename,\n mw.file_size_bytes,\n mw.file_checksum_sha2,\n mw.private_bucket_hash,\n mw.maybe_private_bucket_prefix,\n mw.maybe_private_bucket_extension,\n mw.cached_user_ratings_negative_count,\n mw.cached_user_ratings_positive_count,\n mw.cached_user_ratings_total_count,\n mw.maybe_cached_user_ratings_ratio,\n mw.cached_user_ratings_last_updated_at,\n mw.version,\n mw.created_at,\n mw.updated_at,\n mw.user_deleted_at,\n mw.mod_deleted_at\n FROM model_weights as mw\n JOIN users\n ON users.token = mw.creator_user_token\n WHERE\n users.username = ?\n AND mw.user_deleted_at IS NULL\n AND mw.mod_deleted_at IS NULL\n ",
"query": "\n SELECT\n mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n mw.title,\n mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n mw.maybe_thumbnail_token,\n mw.description_markdown,\n mw.description_rendered_html,\n users.token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_email_gravatar_hash,\n mw.creator_ip_address,\n mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n mw.original_download_url,\n mw.original_filename,\n mw.file_size_bytes,\n mw.file_checksum_sha2,\n mw.public_bucket_hash,\n mw.maybe_public_bucket_prefix,\n mw.maybe_public_bucket_extension,\n mw.cached_user_ratings_negative_count,\n mw.cached_user_ratings_positive_count,\n mw.cached_user_ratings_total_count,\n mw.maybe_cached_user_ratings_ratio,\n mw.cached_user_ratings_last_updated_at,\n mw.version,\n mw.created_at,\n mw.updated_at,\n mw.user_deleted_at,\n mw.mod_deleted_at\n FROM model_weights as mw\n JOIN users\n ON users.token = mw.creator_user_token\n WHERE\n users.username = ?\n AND mw.user_deleted_at IS NULL\n AND mw.mod_deleted_at IS NULL\n ",
"describe": {
"columns": [
{
@@ -185,7 +185,7 @@
},
{
"ordinal": 18,
"name": "private_bucket_hash",
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
@@ -195,7 +195,7 @@
},
{
"ordinal": 19,
"name": "maybe_private_bucket_prefix",
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -205,7 +205,7 @@
},
{
"ordinal": 20,
"name": "maybe_private_bucket_extension",
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -351,5 +351,5 @@
true
]
},
"hash": "a6ed8524fb5339fed17bec6611f8f56b6bdee35bac63c9eacbe21eb032a6a831"
"hash": "854da1bad28fbcf5a8f82c26c52c25a7727957b52b727831ce62518b3c5fc9bd"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\nUPDATE voice_conversion_models\nSET\n maybe_migration_new_model_weights_token = ?\nWHERE token = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "898a2f89d1f3efcb87adaef432cc73117057e44766c330caff6ce2c9d48518fd"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO model_weights\nSET\n token = ?,\n weights_type = ?,\n weights_category = ?,\n title = ?,\n\n maybe_thumbnail_token = NULL,\n maybe_avatar_media_file_token = NULL,\n maybe_cover_media_file_token = NULL,\n description_markdown = NULL,\n description_rendered_html = NULL,\n\n maybe_last_update_user_token = NULL,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n private_bucket_hash = ?,\n maybe_private_bucket_prefix = ?,\n maybe_private_bucket_extension = ?,\n\n cached_user_ratings_total_count = 0,\n cached_user_ratings_positive_count = 0,\n cached_user_ratings_negative_count = 0,\n maybe_cached_user_ratings_ratio = 0.0,\n cached_user_ratings_last_updated_at = NOW(),\n\n maybe_migration_old_model_token = ?\n ",
"query": "\nINSERT INTO model_weights\nSET\n token = ?,\n weights_type = ?,\n weights_category = ?,\n title = ?,\n\n maybe_thumbnail_token = NULL,\n maybe_avatar_media_file_token = NULL,\n maybe_cover_media_file_token = NULL,\n description_markdown = NULL,\n description_rendered_html = NULL,\n\n maybe_last_update_user_token = NULL,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n\n public_bucket_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n cached_user_ratings_total_count = 0,\n cached_user_ratings_positive_count = 0,\n cached_user_ratings_negative_count = 0,\n maybe_cached_user_ratings_ratio = 0.0,\n cached_user_ratings_last_updated_at = NOW(),\n\n maybe_migration_old_model_token = ?\n ",
"describe": {
"columns": [],
"parameters": {
@@ -8,5 +8,5 @@
},
"nullable": []
},
"hash": "685839dd0d875409fc1f3262e234518760c3d5b10c46fb8daf96756746c01c81"
"hash": "8cb3ba5acb8dbef84408370810c98d4b99223761d4547ce7c8a9438ba92e8523"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.private_bucket_hash,\n wt.maybe_private_bucket_prefix,\n wt.maybe_private_bucket_extension,\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n WHERE\n wt.token = ?\n AND wt.user_deleted_at IS NULL\n AND wt.mod_deleted_at IS NULL\n ",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.public_bucket_hash,\n wt.maybe_public_bucket_prefix,\n wt.maybe_public_bucket_extension,\n\n avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,\n avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,\n avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,\n\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n JOIN users\n ON users.token = wt.creator_user_token\n LEFT OUTER JOIN media_files as avatar\n ON avatar.token = wt.maybe_avatar_media_file_token\n WHERE\n wt.token = ?\n AND wt.user_deleted_at IS NULL\n AND wt.mod_deleted_at IS NULL\n ",
"describe": {
"columns": [
{
@@ -75,16 +75,6 @@
},
{
"ordinal": 7,
"name": "creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 8,
"name": "creator_ip_address",
"type_info": {
"type": "VarString",
@@ -94,7 +84,7 @@
}
},
{
"ordinal": 9,
"ordinal": 8,
"name": "creator_set_visibility: enums::common::visibility::Visibility",
"type_info": {
"type": "String",
@@ -103,8 +93,48 @@
"max_size": 28
}
},
{
"ordinal": 9,
"name": "creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 10,
"name": "creator_username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 11,
"name": "creator_display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 12,
"name": "creator_gravatar_hash",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 13,
"name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
@@ -114,7 +144,7 @@
}
},
{
"ordinal": 11,
"ordinal": 14,
"name": "original_download_url",
"type_info": {
"type": "VarString",
@@ -124,7 +154,7 @@
}
},
{
"ordinal": 12,
"ordinal": 15,
"name": "original_filename",
"type_info": {
"type": "VarString",
@@ -134,7 +164,7 @@
}
},
{
"ordinal": 13,
"ordinal": 16,
"name": "file_size_bytes",
"type_info": {
"type": "Long",
@@ -144,7 +174,7 @@
}
},
{
"ordinal": 14,
"ordinal": 17,
"name": "file_checksum_sha2",
"type_info": {
"type": "String",
@@ -154,8 +184,8 @@
}
},
{
"ordinal": 15,
"name": "private_bucket_hash",
"ordinal": 18,
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
@@ -164,8 +194,8 @@
}
},
{
"ordinal": 16,
"name": "maybe_private_bucket_prefix",
"ordinal": 19,
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -174,8 +204,8 @@
}
},
{
"ordinal": 17,
"name": "maybe_private_bucket_extension",
"ordinal": 20,
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -184,7 +214,37 @@
}
},
{
"ordinal": 18,
"ordinal": 21,
"name": "maybe_avatar_public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 22,
"name": "maybe_avatar_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 23,
"name": "maybe_avatar_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 24,
"name": "cached_user_ratings_negative_count",
"type_info": {
"type": "Long",
@@ -194,7 +254,7 @@
}
},
{
"ordinal": 19,
"ordinal": 25,
"name": "cached_user_ratings_positive_count",
"type_info": {
"type": "Long",
@@ -204,7 +264,7 @@
}
},
{
"ordinal": 20,
"ordinal": 26,
"name": "cached_user_ratings_total_count",
"type_info": {
"type": "Long",
@@ -214,7 +274,7 @@
}
},
{
"ordinal": 21,
"ordinal": 27,
"name": "maybe_cached_user_ratings_ratio",
"type_info": {
"type": "Float",
@@ -224,7 +284,7 @@
}
},
{
"ordinal": 22,
"ordinal": 28,
"name": "cached_user_ratings_last_updated_at",
"type_info": {
"type": "Timestamp",
@@ -234,7 +294,7 @@
}
},
{
"ordinal": 23,
"ordinal": 29,
"name": "version",
"type_info": {
"type": "Long",
@@ -244,7 +304,7 @@
}
},
{
"ordinal": 24,
"ordinal": 30,
"name": "created_at",
"type_info": {
"type": "Timestamp",
@@ -254,7 +314,7 @@
}
},
{
"ordinal": 25,
"ordinal": 31,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
@@ -264,7 +324,7 @@
}
},
{
"ordinal": 26,
"ordinal": 32,
"name": "user_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -274,7 +334,7 @@
}
},
{
"ordinal": 27,
"ordinal": 33,
"name": "mod_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -298,6 +358,9 @@
false,
false,
false,
false,
false,
false,
true,
true,
true,
@@ -306,6 +369,9 @@
false,
true,
true,
true,
true,
true,
false,
false,
false,
@@ -318,5 +384,5 @@
true
]
},
"hash": "f0b5767d52c389660d1859e628c746f6f33a9e1c25f21f66aeeb0784df768c8e"
"hash": "93084dc42ae4ada186dc90bdd470122342586f4f19a3ac961286c1c54a5c32b2"
}
@@ -0,0 +1,542 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n id,\n token as `token: tokens::tokens::tts_models::TtsModelToken`,\n\n tts_model_type as `tts_model_type: enums::by_table::tts_models::tts_model_type::TtsModelType`,\n\n text_pipeline_type,\n has_self_contained_vocoder,\n has_self_contained_duration_model,\n has_self_contained_pitch_model,\n text_preprocessing_algorithm,\n\n use_default_mel_multiply_factor,\n maybe_custom_mel_multiply_factor,\n maybe_default_pretrained_vocoder,\n maybe_custom_vocoder_token as `maybe_custom_vocoder_token: tokens::tokens::vocoder_models::VocoderModelToken`,\n\n ietf_language_tag,\n ietf_primary_language_subtag,\n\n title,\n description_markdown,\n description_rendered_html,\n\n voice_name,\n actor_name,\n characteristic,\n version_string,\n\n is_front_page_featured,\n is_twitch_featured,\n\n maybe_suggested_unique_bot_command,\n\n original_download_url,\n original_filename,\n file_size_bytes,\n\n creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n\n creator_ip_address_creation,\n creator_ip_address_last_update,\n\n creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n\n private_bucket_hash,\n private_bucket_object_name,\n\n private_bucket_object_is_archive,\n\n user_ratings_total_count,\n user_ratings_positive_count,\n user_ratings_negative_count,\n\n is_public_listing_approved,\n is_locked_from_user_modification,\n is_locked_from_use,\n\n maybe_mod_comments,\n maybe_mod_user_token as `maybe_mod_user_token: tokens::tokens::users::UserToken`,\n\n maybe_migration_new_model_weights_token as `maybe_migration_new_model_weights_token: tokens::tokens::model_weights::ModelWeightToken`,\n\n version,\n\n created_at,\n updated_at,\n\n user_deleted_at,\n mod_deleted_at\n\nFROM tts_models\nWHERE\n id > ?\nORDER BY id ASC\nLIMIT ?\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": {
"type": "LongLong",
"flags": "NOT_NULL | PRIMARY_KEY | AUTO_INCREMENT",
"char_set": 63,
"max_size": 20
}
},
{
"ordinal": 1,
"name": "token: tokens::tokens::tts_models::TtsModelToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 2,
"name": "tts_model_type: enums::by_table::tts_models::tts_model_type::TtsModelType",
"type_info": {
"type": "String",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
"char_set": 224,
"max_size": 60
}
},
{
"ordinal": 3,
"name": "text_pipeline_type",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 256
}
},
{
"ordinal": 4,
"name": "has_self_contained_vocoder",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 5,
"name": "has_self_contained_duration_model",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 6,
"name": "has_self_contained_pitch_model",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 7,
"name": "text_preprocessing_algorithm",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | ENUM",
"char_set": 224,
"max_size": 28
}
},
{
"ordinal": 8,
"name": "use_default_mel_multiply_factor",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 9,
"name": "maybe_custom_mel_multiply_factor",
"type_info": {
"type": "Double",
"flags": "",
"char_set": 63,
"max_size": 22
}
},
{
"ordinal": 10,
"name": "maybe_default_pretrained_vocoder",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 256
}
},
{
"ordinal": 11,
"name": "maybe_custom_vocoder_token: tokens::tokens::vocoder_models::VocoderModelToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 12,
"name": "ietf_language_tag",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY",
"char_set": 224,
"max_size": 256
}
},
{
"ordinal": 13,
"name": "ietf_primary_language_subtag",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY",
"char_set": 224,
"max_size": 48
}
},
{
"ordinal": 14,
"name": "title",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 15,
"name": "description_markdown",
"type_info": {
"type": "Blob",
"flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 262140
}
},
{
"ordinal": 16,
"name": "description_rendered_html",
"type_info": {
"type": "Blob",
"flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 262140
}
},
{
"ordinal": 17,
"name": "voice_name",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 18,
"name": "actor_name",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 19,
"name": "characteristic",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 20,
"name": "version_string",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 21,
"name": "is_front_page_featured",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 22,
"name": "is_twitch_featured",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 23,
"name": "maybe_suggested_unique_bot_command",
"type_info": {
"type": "VarString",
"flags": "UNIQUE_KEY | BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 24,
"name": "original_download_url",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 2048
}
},
{
"ordinal": 25,
"name": "original_filename",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 26,
"name": "file_size_bytes",
"type_info": {
"type": "Long",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 10
}
},
{
"ordinal": 27,
"name": "creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 28,
"name": "creator_ip_address_creation",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 160
}
},
{
"ordinal": 29,
"name": "creator_ip_address_last_update",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 160
}
},
{
"ordinal": 30,
"name": "creator_set_visibility: enums::common::visibility::Visibility",
"type_info": {
"type": "String",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | ENUM",
"char_set": 224,
"max_size": 28
}
},
{
"ordinal": 31,
"name": "private_bucket_hash",
"type_info": {
"type": "String",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 256
}
},
{
"ordinal": 32,
"name": "private_bucket_object_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 33,
"name": "private_bucket_object_is_archive",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 34,
"name": "user_ratings_total_count",
"type_info": {
"type": "Long",
"flags": "NOT_NULL | UNSIGNED",
"char_set": 63,
"max_size": 10
}
},
{
"ordinal": 35,
"name": "user_ratings_positive_count",
"type_info": {
"type": "Long",
"flags": "NOT_NULL | UNSIGNED",
"char_set": 63,
"max_size": 10
}
},
{
"ordinal": 36,
"name": "user_ratings_negative_count",
"type_info": {
"type": "Long",
"flags": "NOT_NULL | UNSIGNED",
"char_set": 63,
"max_size": 10
}
},
{
"ordinal": 37,
"name": "is_public_listing_approved",
"type_info": {
"type": "Tiny",
"flags": "MULTIPLE_KEY",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 38,
"name": "is_locked_from_user_modification",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL | MULTIPLE_KEY",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 39,
"name": "is_locked_from_use",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL | MULTIPLE_KEY",
"char_set": 63,
"max_size": 1
}
},
{
"ordinal": 40,
"name": "maybe_mod_comments",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 1020
}
},
{
"ordinal": 41,
"name": "maybe_mod_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "MULTIPLE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 42,
"name": "maybe_migration_new_model_weights_token: tokens::tokens::model_weights::ModelWeightToken",
"type_info": {
"type": "VarString",
"flags": "UNIQUE_KEY | BINARY",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 43,
"name": "version",
"type_info": {
"type": "Long",
"flags": "NOT_NULL",
"char_set": 63,
"max_size": 11
}
},
{
"ordinal": 44,
"name": "created_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 45,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
"flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 46,
"name": "user_deleted_at",
"type_info": {
"type": "Timestamp",
"flags": "BINARY",
"char_set": 63,
"max_size": 19
}
},
{
"ordinal": 47,
"name": "mod_deleted_at",
"type_info": {
"type": "Timestamp",
"flags": "BINARY",
"char_set": 63,
"max_size": 19
}
}
],
"parameters": {
"Right": 2
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false,
false,
true,
true,
true,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false,
false,
true,
true,
true,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
true,
true,
true,
false,
false,
false,
true,
true
]
},
"hash": "a44c637e84152864a00f0ba58838c2f3888e3061fc8b44291541840fdb706e44"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO model_weights\nSET\n token = ?,\n weights_type = ?,\n weights_category = ?,\n title = ?,\n maybe_thumbnail_token = NULL,\n maybe_avatar_media_file_token = NULL,\n maybe_cover_media_file_token = NULL,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n maybe_last_update_user_token = NULL,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n\n public_bucket_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n cached_user_ratings_total_count = 0,\n cached_user_ratings_positive_count = 0,\n cached_user_ratings_negative_count = 0,\n maybe_cached_user_ratings_ratio = 0.0,\n cached_user_ratings_last_updated_at = NOW(),\n\n maybe_migration_old_model_token = ?,\n version = ?,\n created_at = ?,\n updated_at = ?,\n user_deleted_at = ?,\n mod_deleted_at = ?\n\nON DUPLICATE KEY UPDATE\n weights_type = ?,\n weights_category = ?,\n title = ?,\n maybe_thumbnail_token = NULL,\n maybe_avatar_media_file_token = NULL,\n maybe_cover_media_file_token = NULL,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n maybe_last_update_user_token = NULL,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n public_bucket_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n cached_user_ratings_total_count = 0,\n cached_user_ratings_positive_count = 0,\n cached_user_ratings_negative_count = 0,\n maybe_cached_user_ratings_ratio = 0.0,\n cached_user_ratings_last_updated_at = NOW(),\n maybe_migration_old_model_token = ?,\n version = ?,\n created_at = ?,\n updated_at = ?,\n user_deleted_at = ?,\n mod_deleted_at = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 43
},
"nullable": []
},
"hash": "aae28dc5fbeff858e4c9dd8b079752ce7892d76b56d2eff95db39c6b6d9f7f11"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\nSELECT\n w.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n w.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n\n w.title,\n vc.has_index_file,\n\n w.private_bucket_hash,\n w.maybe_private_bucket_prefix,\n w.maybe_private_bucket_extension,\n\n w.created_at,\n w.updated_at,\n w.user_deleted_at,\n w.mod_deleted_at\n\nFROM model_weights as w\nJOIN model_weights_extension_voice_conversion_details as vc\nON vc.model_weights_token = w.token\n\nWHERE w.token = ?\n ",
"query": "\nSELECT\n w.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n w.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n\n w.title,\n vc.has_index_file,\n\n w.public_bucket_hash,\n w.maybe_public_bucket_prefix,\n w.maybe_public_bucket_extension,\n\n w.created_at,\n w.updated_at,\n w.user_deleted_at,\n w.mod_deleted_at\n\nFROM model_weights as w\nJOIN model_weights_extension_voice_conversion_details as vc\nON vc.model_weights_token = w.token\n\nWHERE w.token = ?\n ",
"describe": {
"columns": [
{
@@ -45,7 +45,7 @@
},
{
"ordinal": 4,
"name": "private_bucket_hash",
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
@@ -55,7 +55,7 @@
},
{
"ordinal": 5,
"name": "maybe_private_bucket_prefix",
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -65,7 +65,7 @@
},
{
"ordinal": 6,
"name": "maybe_private_bucket_extension",
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -131,5 +131,5 @@
true
]
},
"hash": "0d4c43d9055cea3ce35c71ab8ef27058075414496dc5858d57a9f06d7c076bab"
"hash": "ae34c2ce11250786962be50f167f15f6d2d023afe2bb31994d21a1007d81e0a0"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\n UPDATE model_weights\n SET\n maybe_avatar_media_file_token = ?\n WHERE token = ?\n LIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "bda002b24b98328f0fb0440cfc82a47faf01e5d16d1030e15a7e2da161dd0d80"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n INSERT INTO model_weights\n SET\n token = ?,\n weights_type = ?,\n weights_category = ?,\n title = ?,\n maybe_thumbnail_token = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n maybe_last_update_user_token = ?,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n private_bucket_hash = ?,\n maybe_private_bucket_prefix = ?,\n maybe_private_bucket_extension = ?,\n cached_user_ratings_total_count = ?,\n cached_user_ratings_positive_count = ?,\n cached_user_ratings_negative_count = ?,\n maybe_cached_user_ratings_ratio = ?,\n version = ?\n ",
"query": "\n INSERT INTO model_weights\n SET\n token = ?,\n weights_type = ?,\n weights_category = ?,\n title = ?,\n maybe_thumbnail_token = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n creator_user_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?,\n maybe_last_update_user_token = ?,\n original_download_url = ?,\n original_filename = ?,\n file_size_bytes = ?,\n file_checksum_sha2 = ?,\n public_bucket_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n cached_user_ratings_total_count = ?,\n cached_user_ratings_positive_count = ?,\n cached_user_ratings_negative_count = ?,\n maybe_cached_user_ratings_ratio = ?,\n version = ?\n ",
"describe": {
"columns": [],
"parameters": {
@@ -8,5 +8,5 @@
},
"nullable": []
},
"hash": "274cfb1e3caca706e3dbc3d95064e0f4b354c936f8de948070854920f265cc41"
"hash": "cb07f1522d0d619e2f364af78250080c99e7930ec095c5600abb80a9d8c5154d"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT \n mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n mw.title,\n mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n mw.maybe_thumbnail_token,\n mw.description_markdown,\n mw.description_rendered_html,\n users.token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_email_gravatar_hash,\n mw.creator_ip_address,\n mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n mw.original_download_url,\n mw.original_filename,\n mw.file_size_bytes,\n mw.file_checksum_sha2,\n mw.private_bucket_hash,\n mw.maybe_private_bucket_prefix,\n mw.maybe_private_bucket_extension,\n mw.cached_user_ratings_negative_count,\n mw.cached_user_ratings_positive_count,\n mw.cached_user_ratings_total_count,\n mw.maybe_cached_user_ratings_ratio,\n mw.cached_user_ratings_last_updated_at,\n mw.version,\n mw.created_at,\n mw.updated_at,\n mw.user_deleted_at,\n mw.mod_deleted_at\n FROM model_weights as mw\n JOIN users\n ON users.token = mw.creator_user_token\n WHERE\n users.username = ?\n ",
"query": "\n SELECT \n mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n mw.title,\n mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n mw.maybe_thumbnail_token,\n mw.description_markdown,\n mw.description_rendered_html,\n users.token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash as creator_email_gravatar_hash,\n mw.creator_ip_address,\n mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n mw.original_download_url,\n mw.original_filename,\n mw.file_size_bytes,\n mw.file_checksum_sha2,\n mw.public_bucket_hash,\n mw.maybe_public_bucket_prefix,\n mw.maybe_public_bucket_extension,\n mw.cached_user_ratings_negative_count,\n mw.cached_user_ratings_positive_count,\n mw.cached_user_ratings_total_count,\n mw.maybe_cached_user_ratings_ratio,\n mw.cached_user_ratings_last_updated_at,\n mw.version,\n mw.created_at,\n mw.updated_at,\n mw.user_deleted_at,\n mw.mod_deleted_at\n FROM model_weights as mw\n JOIN users\n ON users.token = mw.creator_user_token\n WHERE\n users.username = ?\n ",
"describe": {
"columns": [
{
@@ -185,7 +185,7 @@
},
{
"ordinal": 18,
"name": "private_bucket_hash",
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
@@ -195,7 +195,7 @@
},
{
"ordinal": 19,
"name": "maybe_private_bucket_prefix",
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -205,7 +205,7 @@
},
{
"ordinal": 20,
"name": "maybe_private_bucket_extension",
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
@@ -351,5 +351,5 @@
true
]
},
"hash": "3d586592422727dcc928574966bdf5081ac69fdbb856fffcbd68187fd99c4642"
"hash": "ec733ae6705dc03fb785e2299c35fcfef23422142735319c8cd091ca8179f97d"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n maybe_origin_model_type = NULL,\n maybe_origin_model_token = NULL,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n\n maybe_creator_file_synthetic_id = ?,\n maybe_creator_category_synthetic_id = ?\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 16
},
"nullable": []
},
"hash": "fed337e365f0bd36cfa543e0ce9d4076cc0bd7809b6c2cb5740151d6583a6d11"
}
Generated
+1
View File
@@ -3528,6 +3528,7 @@ dependencies = [
"easyenv",
"enums",
"errors",
"filesys",
"hashing",
"log",
"my-workspace-hack",
@@ -310,7 +310,7 @@ Secrets:
- Put `.env-secrets` in the root `storyteller-rust` directory.
- Per app secrets (ask Brandon for these):
- Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller-web/config`
- Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller_web/config`
- Put `inference-job.development-secrets.env` in `crates/service/job/inference_job/config`
- Put `donwload-job.development-secrets.env` in `crates/service/job/download_job/config`
@@ -392,4 +392,4 @@ We use this: `https://github.com/juhaku/utoipa`
The documentation example is in:
`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
Write into the api doc to:
`crates/service/web/storyteller_web/src/api_doc.rs`
`crates/service/web/storyteller_web/src/api_doc.rs`
@@ -0,0 +1,12 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE model_weights
RENAME COLUMN public_bucket_hash TO private_bucket_hash;
ALTER TABLE model_weights
RENAME COLUMN maybe_public_bucket_prefix TO maybe_private_bucket_prefix;
ALTER TABLE model_weights
RENAME COLUMN maybe_public_bucket_extension TO maybe_private_bucket_extension;
@@ -0,0 +1,12 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
ALTER TABLE model_weights
RENAME COLUMN private_bucket_hash TO public_bucket_hash;
ALTER TABLE model_weights
RENAME COLUMN maybe_private_bucket_prefix TO maybe_public_bucket_prefix;
ALTER TABLE model_weights
RENAME COLUMN maybe_private_bucket_extension TO maybe_public_bucket_extension;
@@ -113,22 +113,22 @@ CREATE TABLE model_weights (
-- The hash for the bucket directory that contains the original upload
-- as well as any associated other files.
-- The bucket filename for the primary file (not including the directory path) is given by:
-- `[{maybe_private_bucket_prefix}]{private_bucket_hash}[{maybe_private_bucket_extension}]`
private_bucket_hash VARCHAR(32) NOT NULL,
-- `[{maybe_public_bucket_prefix}]{public_bucket_hash}[{maybe_public_bucket_extension}]`
public_bucket_hash VARCHAR(32) NOT NULL,
-- An optional prefix on the bucket filename.
-- If present, this will be prepended to the beginning of the bucket filename to access the file.
-- The bucket filename for the primary file (not including the directory path) is given by:
-- `[{maybe_private_bucket_prefix}]{private_directory_hash}[{maybe_private_bucket_extension}]`
maybe_private_bucket_prefix VARCHAR(16) DEFAULT NULL,
-- `[{maybe_public_bucket_prefix}]{public_bucket_hash}[{maybe_public_bucket_extension}]`
maybe_public_bucket_prefix VARCHAR(16) DEFAULT NULL,
-- An optional appended extension on the bucket filename.
-- If present, this will be appended to the end of the bucket filename to access the file.
-- To allow for flexibility, this extension typically will contain a leading period if
-- the file needs it (eg ".mp4" rather than "mp4")!
-- The bucket filename for the primary file (not including the directory path) is given by:
-- `[{maybe_private_bucket_prefix}]{private_directory_hash}[{maybe_private_bucket_extension}]`
maybe_private_bucket_extension VARCHAR(16) DEFAULT NULL,
-- `[{maybe_public_bucket_prefix}]{public_bucket_hash}[{maybe_public_bucket_extension}]`
maybe_public_bucket_extension VARCHAR(16) DEFAULT NULL,
-- ========== MODERATION DETAILS ==========
@@ -1,49 +0,0 @@
use std::path::PathBuf;
use errors::AnyhowResult;
use errors::bail;
#[deprecated(note = "this is a messy function and should be avoided")]
pub (crate) fn hash_to_bucket_path(file_hash: &str,
optional_root_directory: Option<&str>)
-> AnyhowResult<PathBuf>
{
if file_hash.len() < 4 {
bail!("File length is too short");
}
let root_directory = optional_root_directory
.map(|root| PathBuf::from(root.to_string()))
.unwrap_or(PathBuf::from("/"));
let file_path = format!(
"{}/{}/{}/{}",
&file_hash[0..1],
&file_hash[1..2],
&file_hash[2..3],
&file_hash,
);
Ok(root_directory.join(file_path))
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::util::hash_to_bucket_path::hash_to_bucket_path;
#[test]
fn test_bucket_without_root() {
assert_eq!(
hash_to_bucket_path("abcdefg", None).unwrap(),
PathBuf::from("/a/b/c/abcdefg"));
}
#[test]
fn test_bucket_with_root() {
assert_eq!(
hash_to_bucket_path("abcdefg", Some("/root")).unwrap(),
PathBuf::from("/root/a/b/c/abcdefg"));
}
}
@@ -1,35 +0,0 @@
use errors::anyhow;
use errors::AnyhowResult;
use crate::util::hash_to_bucket_path::hash_to_bucket_path;
#[deprecated(note = "this is a messy function and should be avoided")]
pub fn hash_to_bucket_path_string(file_hash: &str,
optional_root_directory: Option<&str>)
-> AnyhowResult<String>
{
let path = hash_to_bucket_path(file_hash, optional_root_directory)?;
let maybe_path = path.to_str().map(|s| s.to_string());
maybe_path.ok_or(anyhow!("Path could not be converted"))
}
#[cfg(test)]
mod tests {
use crate::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
#[test]
fn test_bucket_without_root() {
assert_eq!(
hash_to_bucket_path_string("abcdefg", None).unwrap(),
"/a/b/c/abcdefg".to_string());
}
#[test]
fn test_bucket_with_root() {
assert_eq!(
hash_to_bucket_path_string("abcdefg", Some("/root")).unwrap(),
"/root/a/b/c/abcdefg".to_string());
}
}
@@ -5,6 +5,7 @@
/// With a million items, each directory will have ~244 items.
/// With a billion items, each directory will have ~244,140 items (impractical at this scale).
///
#[deprecated(note = "Use hashed_directory_path_long_string instead! (This function is only used by BucketPathUnifier, which should also die)")]
pub fn hashed_directory_path_short_string(file_hash: &str) -> String {
match file_hash.len() {
0 | 1 => "".to_string(),
-2
View File
@@ -1,4 +1,2 @@
pub (crate) mod hash_to_bucket_path;
pub mod hash_to_bucket_path_string;
pub mod hashed_directory_path_long_string;
pub mod hashed_directory_path_short_string;
@@ -49,6 +49,10 @@ pub enum IdCategory {
#[serde(rename = "model_weights")]
ModelWeights,
/// Files that are uploaded with no general product area they belong to. (Eg. local dev testing)
#[serde(rename = "file_upload")]
FileUpload,
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -68,6 +72,7 @@ impl IdCategory {
Self::ZeroShotTtsResult => "zs_tts_result",
Self::VideoFilterResult => "video_filter",
Self::ModelWeights => "model_weights",
Self::FileUpload => "file_upload",
}
}
@@ -82,6 +87,7 @@ impl IdCategory {
"zs_tts_result" => Ok(Self::ZeroShotTtsResult),
"video_filter" => Ok(Self::VideoFilterResult),
"model_weights" => Ok(Self::ModelWeights),
"file_upload" => Ok(Self::FileUpload),
_ => Err(format!("invalid value: {:?}", value)),
}
}
@@ -99,6 +105,7 @@ impl IdCategory {
Self::ZeroShotVoiceDataset,
Self::ZeroShotVoiceEmbedding,
Self::ModelWeights,
Self::FileUpload,
])
}
}
@@ -121,7 +128,8 @@ mod tests {
assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset");
assert_serialization(IdCategory::ZeroShotVoiceEmbedding, "zs_voice");
assert_serialization(IdCategory::ZeroShotTtsResult, "zs_tts_result");
assert_serialization(IdCategory::ModelWeights, "model_weights")
assert_serialization(IdCategory::ModelWeights, "model_weights");
assert_serialization(IdCategory::FileUpload, "file_upload");
}
#[test]
@@ -135,6 +143,7 @@ mod tests {
assert_eq!(IdCategory::ZeroShotVoiceEmbedding.to_str(), "zs_voice");
assert_eq!(IdCategory::ZeroShotTtsResult.to_str(), "zs_tts_result");
assert_eq!(IdCategory::ModelWeights.to_str(), "model_weights");
assert_eq!(IdCategory::FileUpload.to_str(), "file_upload");
}
#[test]
@@ -148,13 +157,14 @@ mod tests {
assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZeroShotVoiceEmbedding);
assert_eq!(IdCategory::from_str("zs_tts_result").unwrap(), IdCategory::ZeroShotTtsResult);
assert_eq!(IdCategory::from_str("model_weights").unwrap(), IdCategory::ModelWeights);
assert_eq!(IdCategory::from_str("file_upload").unwrap(), IdCategory::FileUpload);
}
#[test]
fn all_variants() {
// Static check
let mut variants = IdCategory::all_variants();
assert_eq!(variants.len(), 9);
assert_eq!(variants.len(), 10);
assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile));
assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult));
assert_eq!(variants.pop_first(), Some(IdCategory::VideoFilterResult));
@@ -164,6 +174,7 @@ mod tests {
assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotVoiceDataset));
assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotVoiceEmbedding));
assert_eq!(variants.pop_first(), Some(IdCategory::ModelWeights));
assert_eq!(variants.pop_first(), Some(IdCategory::FileUpload));
assert_eq!(variants.pop_first(), None);
// Generated check
@@ -6,12 +6,13 @@ use serde::Serialize;
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Used in the `media_files` table in a `VARCHAR` field.
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum MediaFileOriginCategory {
/// ML model inference output - uploaded models or zero shot.
@@ -6,12 +6,13 @@ use serde::Serialize;
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Used in the `media_files` table in a `VARCHAR` field.
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum MediaFileOriginModelType {
/// RVC (v2) voice conversion models
@@ -4,6 +4,7 @@ use std::collections::BTreeSet;
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Used in the `media_files` table in `VARCHAR(16)` field `origin_product_category`.
///
@@ -14,7 +15,7 @@ use strum::EnumIter;
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
pub enum MediaFileOriginProductCategory {
/// Media files created by (or uploaded for) the Face Animator product.
/// The underlying model could be SadTalker, Wav2Lip, or some future model
@@ -4,12 +4,13 @@ use std::collections::BTreeSet;
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Used in the `media_files` table in a `VARCHAR` field.
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
#[serde(rename_all = "lowercase")]
pub enum MediaFileType {
/// Audio files: wav, mp3, etc.
@@ -4,10 +4,11 @@ use serde::Serialize;
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Used in the `user_bookmarks` table in a `VARCHAR(32)` field named `entity_type`.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize, ToSchema)]
pub enum UserBookmarkEntityType {
/// User
#[serde(rename = "user")]
@@ -2,11 +2,12 @@ use std::fmt::Debug;
use serde::Deserialize;
use serde::Serialize;
use utoipa::ToSchema;
use crate::prefixes::TokenPrefix;
/// The primary key for Audit Logs.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
#[sqlx(transparent)]
pub struct MediaFileToken(pub String);
@@ -2,11 +2,12 @@ use std::fmt::Debug;
use serde::Deserialize;
use serde::Serialize;
use utoipa::ToSchema;
use crate::prefixes::TokenPrefix;
/// The primary key for Audit Logs.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
#[sqlx(transparent)]
pub struct UserBookmarkToken(pub String);
@@ -9,7 +9,7 @@ use enums::by_table::model_weights::{
};
use enums::common::visibility::Visibility;
use errors::{anyhow, AnyhowResult};
use mysql_queries::queries::model_weights::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::model_weights::create::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
@@ -335,9 +335,9 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
original_filename: Some(original_filename),
file_size_bytes: 1024,
file_checksum_sha2: "checksum1".to_string(),
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some(private_bucket_prefix),
maybe_private_bucket_extension: Some(private_bucket_extension),
public_bucket_hash: "bucket_hash1".to_string(),
maybe_public_bucket_prefix: Some(private_bucket_prefix),
maybe_public_bucket_extension: Some(private_bucket_extension),
cached_user_ratings_total_count,
cached_user_ratings_positive_count,
cached_user_ratings_negative_count,
@@ -381,9 +381,9 @@ pub async fn seed_weights_for_user_token(
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum1".to_string(),
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: "bucket_hash1".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -407,9 +407,9 @@ pub async fn seed_weights_for_user_token(
original_filename: Some("filename2.txt".to_string()),
file_size_bytes: 2024,
file_checksum_sha2: "checksum2".to_string(),
private_bucket_hash: "bucket_hash2".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("tt2".to_string()),
public_bucket_hash: "bucket_hash2".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("tt2".to_string()),
cached_user_ratings_total_count: 20,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 2,
@@ -433,9 +433,9 @@ pub async fn seed_weights_for_user_token(
original_filename: Some("filename3.txt".to_string()),
file_size_bytes: 3024,
file_checksum_sha2: "checksum3".to_string(),
private_bucket_hash: "bucket_hash3".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("safetensors".to_string()),
public_bucket_hash: "bucket_hash3".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("safetensors".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -459,9 +459,9 @@ pub async fn seed_weights_for_user_token(
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum4".to_string(),
private_bucket_hash: "bucket_hash4".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("LoRA".to_string()),
public_bucket_hash: "bucket_hash4".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("LoRA".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -485,9 +485,9 @@ pub async fn seed_weights_for_user_token(
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum4".to_string(),
private_bucket_hash: "bucket_hash4".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("LoRA".to_string()),
public_bucket_hash: "bucket_hash4".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("LoRA".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -537,9 +537,9 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>,user_token: UserToke
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum1".to_string(),
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: "bucket_hash1".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -563,9 +563,9 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>,user_token: UserToke
original_filename: Some("filename2.txt".to_string()),
file_size_bytes: 2024,
file_checksum_sha2: "checksum2".to_string(),
private_bucket_hash: "bucket_hash2".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("tt2".to_string()),
public_bucket_hash: "bucket_hash2".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("tt2".to_string()),
cached_user_ratings_total_count: 20,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 2,
@@ -589,9 +589,9 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>,user_token: UserToke
original_filename: Some("filename3.txt".to_string()),
file_size_bytes: 3024,
file_checksum_sha2: "checksum3".to_string(),
private_bucket_hash: "bucket_hash3".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("safetensors".to_string()),
public_bucket_hash: "bucket_hash3".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("safetensors".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -615,9 +615,9 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>,user_token: UserToke
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum4".to_string(),
private_bucket_hash: "bucket_hash4".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("LoRA".to_string()),
public_bucket_hash: "bucket_hash4".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("LoRA".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -641,9 +641,9 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>,user_token: UserToke
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1025,
file_checksum_sha2: "checksum5".to_string(),
private_bucket_hash: "bucket_hash5".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("LoRA".to_string()),
public_bucket_hash: "bucket_hash5".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("LoRA".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -15,8 +15,8 @@ use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::queries::media_files::create::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use mysql_queries::queries::media_files::get_media_file::get_media_file;
use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use storyteller_root::get_seed_tool_data_root;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::users::UserToken;
@@ -15,7 +15,7 @@ use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mysql_queries::queries::model_weights::create_weight::{
use mysql_queries::queries::model_weights::create::create_weight::{
create_weight,
CreateModelWeightsArgs,
};
@@ -141,9 +141,9 @@ pub async fn seed_weights_files(
original_filename: Some(original_filename.to_string()),
file_size_bytes: file_size_bytes as i32, // Assuming the file size is 0
file_checksum_sha2: sha256_checksum.to_string(), // Assuming the file checksum is an empty string
private_bucket_hash: bucket_hash,
maybe_private_bucket_prefix: Some(maybe_bucket_prefix.to_string()),
maybe_private_bucket_extension: Some(maybe_bucket_extension.to_string()),
public_bucket_hash: bucket_hash,
maybe_public_bucket_prefix: Some(maybe_bucket_prefix.to_string()),
maybe_public_bucket_extension: Some(maybe_bucket_extension.to_string()),
cached_user_ratings_total_count,
cached_user_ratings_positive_count,
cached_user_ratings_negative_count,
+1 -1
View File
@@ -11,7 +11,7 @@ use enums::common::visibility::Visibility;
use errors::{AnyhowResult, bail};
use filesys::file_read_bytes::file_read_bytes;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use crate::cli_args::parse_cli_args;
+1
View File
@@ -20,6 +20,7 @@ config = { path = "../../lib/deprecated/config" }
easyenv = { path = "../../lib/easyenv" }
enums = { path = "../../_types/enums" }
errors = { path = "../../lib/errors" }
filesys = { path = "../../lib/files/filesys" }
hashing = { path = "../../lib/files/hashing" }
mysql_queries = { path = "../../lib/mysql_queries" }
storyteller_root = { path = "../../lib/storyteller_root" }
@@ -4,6 +4,8 @@ use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
use errors::{anyhow, AnyhowResult};
use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
use filesys::safe_delete_temp_file::safe_delete_temp_file;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mysql_queries::queries::model_weights::migration::upsert_model_weight_from_voice_conversion_model::CopiedFileData;
use mysql_queries::queries::voice_conversion::migration::list_whole_voice_conversion_models_using_cursor::WholeVoiceConversionModelRecord;
@@ -43,11 +45,14 @@ async fn copy_model(model: &WholeVoiceConversionModelRecord, deps: &Deps) -> Any
VoiceConversionModelType::SoftVc => return Err(anyhow!("we never built softvc models")),
};
deps.bucket_development_public.upload_filename_with_content_type(
deps.bucket_production_public.upload_filename_with_content_type(
&new_model_bucket_path.get_full_object_path_str(),
&model_temp_fs_path,
"application/octet-stream").await?;
safe_delete_temp_file(&model_temp_fs_path);
safe_delete_temp_directory(&temp_dir);
Ok(CopiedFileData {
bucket_path: new_model_bucket_path,
file_sha_hash: file_checksum,
@@ -68,10 +73,13 @@ async fn copy_index_file(model: &WholeVoiceConversionModelRecord, deps: &Deps, b
let new_model_bucket_path =
WeightFileBucketPath::rvc_index_file_from_object_hash(bucket_path.get_object_hash());
deps.bucket_development_public.upload_filename_with_content_type(
deps.bucket_production_public.upload_filename_with_content_type(
&new_model_bucket_path.get_full_object_path_str(),
&model_temp_fs_path,
"application/octet-stream").await?;
safe_delete_temp_file(&model_temp_fs_path);
safe_delete_temp_directory(&temp_dir);
Ok(()) // NB: We don't care about the path of the index file.
}
@@ -1,9 +1,15 @@
use std::thread;
use std::time::Duration;
use log::{info, warn};
use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
use enums::by_table::model_weights::weights_types::WeightsType;
use errors::AnyhowResult;
use mysql_queries::queries::model_weights::migration::upsert_model_weight_from_voice_conversion_model::upsert_model_weight_from_voice_conversion_model;
use mysql_queries::queries::voice_conversion::migration::list_whole_voice_conversion_models_using_cursor::list_whole_voice_conversion_models_using_cursor;
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use mysql_queries::queries::model_weights::migration::upsert_model_weight_from_voice_conversion_model::{CopiedFileData, upsert_model_weight_from_voice_conversion_model};
use mysql_queries::queries::voice_conversion::migration::list_whole_voice_conversion_models_using_cursor::{list_whole_voice_conversion_models_using_cursor, WholeVoiceConversionModelRecord};
use tokens::tokens::model_weights::ModelWeightToken;
use crate::deps::Deps;
use crate::migrations::voice_conversion_to_weights::copy_cloud_files::copy_cloud_files;
@@ -15,17 +21,19 @@ pub async fn migrate_voice_conversion_to_weights(deps: &Deps) -> AnyhowResult<()
let mut cursor = 0;
loop {
info!("Querying {PAGE_SIZE} models at cursor = {cursor}");
let results
= list_whole_voice_conversion_models_using_cursor(&deps.mysql_production, PAGE_SIZE, cursor).await?;
if results.is_empty() {
warn!("No more results found; exiting.");
break;
}
for result in results.iter() {
println!("result: {:?}\n\n", result);
let copied_data = copy_cloud_files(result, &deps).await?;
upsert_model_weight_from_voice_conversion_model(result, &deps.mysql_development, &copied_data).await?;
println!("\n\nmigrating over result {:?} : {:?} ...", result.token, result);
copy_model_record_and_bucket_files(result, &deps).await?;
}
if let Some(last_id) = results.last().map(|result| result.id) {
@@ -37,3 +45,48 @@ pub async fn migrate_voice_conversion_to_weights(deps: &Deps) -> AnyhowResult<()
Ok(())
}
async fn copy_model_record_and_bucket_files(record: &WholeVoiceConversionModelRecord, deps: &Deps) -> AnyhowResult<()> {
let mut maybe_copied_data = None;
match &record.maybe_migration_new_model_weights_token {
None => {
info!("Copying bucket files for record {:?} ...", record.token);
let result = copy_cloud_files(record, &deps).await?;
maybe_copied_data = Some(result);
}
Some(token) => {
info!("Existing migrated record; skipping cloud bucket copy...");
maybe_copied_data = lookup_bucket_details_from_record(token, &deps).await?;
}
}
if let Some(copied_data) = &maybe_copied_data {
info!("Upserting records...");
upsert_model_weight_from_voice_conversion_model(record, &deps.mysql_production, copied_data).await?;
}
Ok(())
}
/// Look up the bucket storage details from a previously-migrated record
async fn lookup_bucket_details_from_record(model_weight_token: &ModelWeightToken, deps: &Deps) -> AnyhowResult<Option<CopiedFileData>> {
let maybe_model_weight_record =
get_weight_by_token(model_weight_token, true, &deps.mysql_production).await?;
let model_weight_record = match maybe_model_weight_record {
None => return Ok(None),
Some(record) => record,
};
let bucket_path = match model_weight_record.weights_type {
WeightsType::RvcV2 => WeightFileBucketPath::rvc_model_file_from_object_hash(&model_weight_record.public_bucket_hash),
WeightsType::SoVitsSvc => WeightFileBucketPath::svc_model_file_from_object_hash(&model_weight_record.public_bucket_hash),
_ => return Ok(None), // NB: Stable Diffusion, etc. aren't valid for migration
};
Ok(Some(CopiedFileData {
bucket_path,
file_sha_hash: model_weight_record.file_checksum_sha2,
}))
}
@@ -6,28 +6,54 @@ use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::model_weights::weights_types::WeightsType;
use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
use errors::AnyhowResult;
use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
use mysql_queries::queries::model_weights::inference::get_model_weight_for_voice_conversion_inference::{get_model_weight_for_voice_conversion_inference, ModelWeightForVoiceConversionInference};
use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference};
use mysql_queries::queries::model_weights::inference::get_model_weight_for_voice_conversion_inference::{get_model_weight_for_voice_conversion_inference, ModelWeightError, ModelWeightForVoiceConversionInference};
use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference, VoiceConversionModelForInferenceError};
use tokens::tokens::model_weights::ModelWeightToken;
#[derive(Clone, Debug)]
pub enum VcModelError {
ModelDeleted,
DatabaseError { reason: String },
}
impl std::fmt::Display for VcModelError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VcModelError::ModelDeleted => write!(f, "ModelDeleted"),
VcModelError::DatabaseError { reason} =>
write!(f, "Database error: {:?}", reason),
}
}
}
impl std::error::Error for VcModelError {}
/// Query model info
/// Depending on the token prefix, this does either a modern or legacy lookup.
pub async fn query_vc_model_for_migration(model_token: &str, mysql_pool: &MySqlPool) -> AnyhowResult<Option<VcModel>> {
pub async fn query_vc_model_for_migration(model_token: &str, mysql_pool: &MySqlPool) -> Result<Option<VcModel>, VcModelError> {
// NB: This is temporary migration code as we switch from the `voice_conversion_models` table to the `model_weights` table.
if model_token.starts_with(ModelWeightToken::token_prefix()) {
let model_weights_token = ModelWeightToken::new_from_str(model_token);
let maybe_model = get_model_weight_for_voice_conversion_inference(
&mysql_pool, &model_weights_token).await?;
&mysql_pool, &model_weights_token).await.map_err(|err|
match err {
ModelWeightError::ModelDeleted => VcModelError::ModelDeleted,
ModelWeightError::DatabaseError { reason } => VcModelError::DatabaseError { reason },
}
)?;
Ok(maybe_model
.map(|model| VcModel::ModelWeight(model)))
} else {
let maybe_vc_model = get_voice_conversion_model_for_inference(
&mysql_pool, model_token).await?;
&mysql_pool, model_token).await.map_err(|err|
match err {
VoiceConversionModelForInferenceError::ModelDeleted => VcModelError::ModelDeleted,
VoiceConversionModelForInferenceError::DatabaseError { reason } => VcModelError::DatabaseError { reason },
})?;
Ok(maybe_vc_model
.map(|model| VcModel::LegacyVoiceConversion(model)))
@@ -361,7 +387,7 @@ mod tests {
model.weights_type = WeightsType::RvcV2;
model.has_index_file = true;
let path = VcModel::ModelWeight(model).get_index_file_cloud_bucket_path(&BucketPathUnifier::default_paths());
assert_eq!(path, Some(PathBuf::from("/weights/h/a/s/hash/hash.rvc_index")));
assert_eq!(path, Some(PathBuf::from("/weights/h/a/s/hash/rvc_hash.index")));
}
}
}
@@ -52,7 +52,7 @@ SELECT
id,
token,
creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
creator_user_token,
creator_ip_address,
creator_set_visibility,
@@ -101,7 +101,7 @@ WHERE
id: GenericDownloadJobId(record.id),
download_job_token: DownloadJobToken::new(record.token),
creator_ip_address: record.creator_ip_address,
creator_user_token: record.creator_user_token,
creator_user_token: UserToken::new_from_str(&record.creator_user_token),
// NB: Failure case for parsing visibility - default to private
creator_set_visibility: Visibility::from_str(&record.creator_set_visibility)
.unwrap_or(Visibility::Private),
@@ -144,7 +144,7 @@ struct AvailableDownloadJobRawInternal {
pub id: i64,
pub token: String,
pub creator_user_token: UserToken,
pub creator_user_token: String,
pub creator_ip_address: String,
pub creator_set_visibility: String,
@@ -0,0 +1,158 @@
use anyhow::anyhow;
use sqlx;
use sqlx::MySqlPool;
use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::users::UserToken;
use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id;
pub enum UploadType {
Filesystem,
DeviceCaptureApi,
}
pub struct InsertMediaFileFromUploadArgs<'a> {
pub pool: &'a MySqlPool,
pub maybe_creator_user_token: Option<&'a UserToken>,
pub maybe_creator_anonymous_visitor_token: Option<&'a AnonymousVisitorTrackingToken>,
pub creator_ip_address: &'a str,
pub creator_set_visibility: Visibility,
pub upload_type: UploadType,
pub media_file_type: MediaFileType,
pub maybe_mime_type: Option<&'a str>,
pub file_size_bytes: u64,
pub duration_millis: u64,
pub sha256_checksum: &'a str,
// TODO: Media duration.
//pub duration_millis: u64,
pub public_bucket_directory_hash: &'a str,
pub maybe_public_bucket_prefix: Option<&'a str>,
pub maybe_public_bucket_extension: Option<&'a str>,
}
pub async fn insert_media_file_from_file_upload(
args: InsertMediaFileFromUploadArgs<'_>
) -> AnyhowResult<(MediaFileToken, u64)>
{
let token = MediaFileToken::generate();
let origin_category = match args.upload_type {
UploadType::Filesystem => MediaFileOriginCategory::Upload,
UploadType::DeviceCaptureApi => MediaFileOriginCategory::DeviceApi,
};
let mut maybe_creator_file_synthetic_id : Option<u64> = None;
let mut maybe_creator_category_synthetic_id : Option<u64> = None;
let mut transaction = args.pool.begin().await?;
if let Some(user_token) = args.maybe_creator_user_token.as_deref() {
let next_media_file_id = transactional_increment_generic_synthetic_id(
&user_token,
IdCategory::MediaFile,
&mut transaction
).await?;
let next_voice_conversion_id = transactional_increment_generic_synthetic_id(
&user_token,
IdCategory::FileUpload,
&mut transaction
).await?;
maybe_creator_file_synthetic_id = Some(next_media_file_id);
maybe_creator_category_synthetic_id = Some(next_voice_conversion_id);
}
const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Unknown;
let record_id = {
let query_result = sqlx::query!(
r#"
INSERT INTO media_files
SET
token = ?,
origin_category = ?,
origin_product_category = ?,
maybe_origin_model_type = NULL,
maybe_origin_model_token = NULL,
media_type = ?,
maybe_mime_type = ?,
file_size_bytes = ?,
checksum_sha2 = ?,
public_bucket_directory_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
maybe_creator_user_token = ?,
maybe_creator_anonymous_visitor_token = ?,
creator_ip_address = ?,
creator_set_visibility = ?,
maybe_creator_file_synthetic_id = ?,
maybe_creator_category_synthetic_id = ?
"#,
token.as_str(),
origin_category.to_str(),
ORIGIN_PRODUCT_CATEGORY.to_str(),
args.media_file_type.to_str(),
args.maybe_mime_type,
args.file_size_bytes,
args.sha256_checksum,
args.public_bucket_directory_hash,
args.maybe_public_bucket_prefix,
args.maybe_public_bucket_extension,
args.maybe_creator_user_token,
args.maybe_creator_anonymous_visitor_token,
args.creator_ip_address,
args.creator_set_visibility.to_str(),
maybe_creator_file_synthetic_id,
maybe_creator_category_synthetic_id,
)
.execute(&mut *transaction)
.await;
let record_id = match query_result {
Ok(res) => {
res.last_insert_id()
},
Err(err) => {
// TODO: handle better
//transaction.rollback().await?;
return Err(anyhow!("Mysql error: {:?}", err));
}
};
record_id
};
transaction.commit().await?;
Ok((token, record_id))
}
@@ -0,0 +1,7 @@
pub mod insert_media_file_from_cli_tool;
pub mod insert_media_file_from_face_animation;
pub mod insert_media_file_from_file_upload;
pub mod insert_media_file_from_rerender;
pub mod insert_media_file_from_tacotron2;
pub mod insert_media_file_from_voice_conversion;
pub mod insert_media_file_from_zero_shot_tts;
@@ -44,6 +44,9 @@ pub struct MediaFileListItem {
pub creator_set_visibility: Visibility,
pub comment_count: u64,
pub favorite_count: u64,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -100,6 +103,8 @@ pub async fn list_media_files(args: ListMediaFilesArgs<'_>) -> AnyhowResult<Medi
maybe_creator_display_name: record.maybe_creator_display_name,
maybe_creator_gravatar_hash: record.maybe_creator_gravatar_hash,
creator_set_visibility: record.creator_set_visibility,
comment_count: record.comment_count as u64,
favorite_count: record.favorite_count as u64,
created_at: record.created_at,
updated_at: record.updated_at,
}
@@ -148,14 +153,22 @@ SELECT
m.creator_set_visibility,
m.created_at,
m.updated_at
m.updated_at,
COUNT(f.id) as favorite_count,
COUNT(c.id) as comment_count
FROM media_files AS m
LEFT OUTER JOIN users AS u
ON m.maybe_creator_user_token = u.token
WHERE m.user_deleted_at IS NULL
AND m.mod_deleted_at IS NULL
LEFT OUTER JOIN favorites as f
ON f.entity_type = 'media_file' AND f.entity_token = m.token
LEFT OUTER JOIN comments as c
ON c.entity_type = 'media_file' AND c.entity_token = c.token
WHERE
m.user_deleted_at IS NULL
AND m.mod_deleted_at IS NULL
"#
);
@@ -186,6 +199,9 @@ AND m.mod_deleted_at IS NULL
}
}
query_builder.push(" GROUP BY m.id");
if cursor_is_reversed {
query_builder.push(" ORDER BY m.id ASC ");
} else {
@@ -218,6 +234,9 @@ struct MediaFileListItemInternal {
creator_set_visibility: Visibility,
comment_count: i64,
favorite_count: i64,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
}
@@ -257,6 +276,8 @@ impl FromRow<'_, MySqlRow> for MediaFileListItemInternal {
creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
comment_count: row.try_get("comment_count")?,
favorite_count: row.try_get("favorite_count")?,
})
}
}
@@ -51,13 +51,12 @@ pub struct ListMediaFileForUserArgs<'a> {
pub maybe_filter_media_type: Option<MediaFileType>,
pub page_size: usize,
pub page_index: usize,
pub cursor_is_reversed: bool,
pub sort_ascending: bool,
pub view_as: ViewAs,
pub mysql_pool: &'a MySqlPool,
}
pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult<MediaFileListPage> {
/// Let's figure out how many results we could have returned total
let count_fields = select_total_count_field();
let mut count_query_builder = query_builder(
@@ -66,16 +65,13 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An
false,
0,
0,
args.cursor_is_reversed,
args.sort_ascending,
args.view_as,
count_fields.as_str(),
);
let row_count_query = count_query_builder.build_query_scalar::<i64>();
let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?;
/// Figure out limit start and end based on page size and indexes as requested
let limit_start = args.page_size * args.page_index;
let limit_end = limit_start + args.page_size;
/// Now fetch the actual results with all the fields
let result_fields = select_result_fields();
@@ -83,9 +79,9 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An
args.maybe_filter_media_type,
args.username,
true,
limit_start,
limit_end,
args.cursor_is_reversed,
args.page_index,
args.page_size,
args.sort_ascending,
args.view_as,
result_fields.as_str(),
);
@@ -115,7 +111,7 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An
Ok(MediaFileListPage {
records: results,
sort_ascending: !args.cursor_is_reversed,
sort_ascending: args.sort_ascending,
current_page: args.page_index,
total_page_count: number_of_pages,
})
@@ -157,9 +153,9 @@ fn query_builder<'a>(
maybe_filter_media_type: Option<MediaFileType>,
username: &'a str,
enforce_limits: bool,
limit_start: usize,
limit_end: usize,
cursor_is_reversed: bool,
page_index: usize,
page_size: usize,
sort_ascending: bool,
view_as: ViewAs,
select_fields: &'a str,
) -> QueryBuilder<'a, MySql> {
@@ -201,18 +197,15 @@ WHERE m.user_deleted_at IS NULL
}
}
if cursor_is_reversed {
query_builder.push(" ORDER BY m.id ASC ");
if sort_ascending {
query_builder.push(" ORDER BY m.created_at ASC ");
} else {
query_builder.push(" ORDER BY m.id DESC ");
query_builder.push(" ORDER BY m.created_at DESC ");
}
if enforce_limits {
if cursor_is_reversed {
query_builder.push(format!(" LIMIT {limit_start}, {limit_end} "));
} else {
query_builder.push(format!(" LIMIT {limit_end}, {limit_start} "));
}
let offset = page_index * page_size;
query_builder.push(format!(" LIMIT {page_size} OFFSET {offset} "));
}
query_builder
@@ -0,0 +1,3 @@
pub mod list_media_files;
pub mod list_media_files_by_tokens;
pub mod list_media_files_for_user;
@@ -1,12 +1,5 @@
pub mod create;
pub mod delete_media_file;
pub mod get_media_file;
pub mod insert_media_file_from_cli_tool;
pub mod insert_media_file_from_face_animation;
pub mod insert_media_file_from_rerender;
pub mod insert_media_file_from_tacotron2;
pub mod insert_media_file_from_voice_conversion;
pub mod insert_media_file_from_zero_shot_tts;
pub mod list_media_files;
pub mod list_media_files_by_tokens;
pub mod list_media_files_for_user;
pub mod list;
pub mod update_media_file;
@@ -79,9 +79,9 @@ SET
creator_ip_address = ?,
creator_set_visibility = ?,
private_bucket_hash = ?,
maybe_private_bucket_prefix = ?,
maybe_private_bucket_extension = ?,
public_bucket_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
cached_user_ratings_total_count = 0,
cached_user_ratings_positive_count = 0,
@@ -24,9 +24,9 @@ pub struct CreateModelWeightsArgs<'a> {
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_negative_count: u32,
@@ -58,9 +58,9 @@ pub async fn create_weight(args: CreateModelWeightsArgs<'_>) -> AnyhowResult<Mod
original_filename = ?,
file_size_bytes = ?,
file_checksum_sha2 = ?,
private_bucket_hash = ?,
maybe_private_bucket_prefix = ?,
maybe_private_bucket_extension = ?,
public_bucket_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
cached_user_ratings_total_count = ?,
cached_user_ratings_positive_count = ?,
cached_user_ratings_negative_count = ?,
@@ -82,9 +82,9 @@ pub async fn create_weight(args: CreateModelWeightsArgs<'_>) -> AnyhowResult<Mod
args.original_filename,
args.file_size_bytes,
args.file_checksum_sha2,
args.private_bucket_hash,
args.maybe_private_bucket_prefix,
args.maybe_private_bucket_extension,
args.public_bucket_hash,
args.maybe_public_bucket_prefix,
args.maybe_public_bucket_extension,
args.cached_user_ratings_total_count,
args.cached_user_ratings_positive_count,
args.cached_user_ratings_negative_count,
@@ -1 +1,2 @@
pub mod create_model_weight_from_voice_conversion_download_job;
pub mod create_model_weight_from_voice_conversion_download_job;
pub mod create_weight;
@@ -0,0 +1,2 @@
pub mod set_model_weight_avatar;
pub mod update_weight;
@@ -0,0 +1,32 @@
use sqlx::MySqlPool;
use errors::AnyhowResult;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
pub struct UpdateArgs<'a> {
pub model_weight_token: &'a ModelWeightToken,
pub maybe_avatar_media_file_token: Option<&'a MediaFileToken>,
//pub update_ip_address: &'a str,
pub mysql_pool: &'a MySqlPool
}
pub async fn set_model_weight_avatar(args: UpdateArgs<'_>) -> AnyhowResult<()>{
let transaction = args.mysql_pool.begin().await?;
let query_result = sqlx::query!(
r#"
UPDATE model_weights
SET
maybe_avatar_media_file_token = ?
WHERE token = ?
LIMIT 1
"#,
args.maybe_avatar_media_file_token,
args.model_weight_token,
).execute(args.mysql_pool).await;
transaction.commit().await?;
Ok(())
}
@@ -1,22 +1,20 @@
use container_common::anyhow_result::AnyhowResult;
use sqlx::{ MySqlPool, MySql };
use sqlx::pool::PoolConnection;
use anyhow::anyhow;
use log::{ error };
use chrono::{DateTime, Utc};
use log::error;
use sqlx::{MySql, MySqlPool};
use sqlx::pool::PoolConnection;
use enums::by_table::model_weights::weights_types::WeightsType;
use container_common::anyhow_result::AnyhowResult;
use enums::by_table::model_weights::weights_category::WeightsCategory;
use enums::by_table::model_weights::weights_types::WeightsType;
use enums::common::visibility::Visibility;
use tokens::tokens::{ users::UserToken, model_weights::ModelWeightToken };
use chrono::{ DateTime, Utc };
use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
// Notes ensure that Enums have sqlx::Type
// Notes ensure that Enums have sqlx::Type
// 'weights_type: enums::by_table::model_weights::weights_types::WeightsType' use this to map
// Retrieved Model Weight can be constrained to the fields that are needed
pub struct RetrivedModelWeight {
pub struct RetrievedModelWeight {
pub token: ModelWeightToken,
pub title: String,
pub weights_type: WeightsType,
@@ -24,17 +22,28 @@ pub struct RetrivedModelWeight {
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_username: String,
pub creator_display_name: String,
pub creator_gravatar_hash: String,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>, // TODO crossproduct rename these to public thx ^-^
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub maybe_avatar_public_bucket_hash: Option<String>,
pub maybe_avatar_public_bucket_prefix: Option<String>,
pub maybe_avatar_public_bucket_extension: Option<String>,
pub cached_user_ratings_negative_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_total_count: u32,
@@ -51,7 +60,7 @@ pub async fn get_weight_by_token(
weight_token: &ModelWeightToken,
can_see_deleted: bool,
mysql_pool: &MySqlPool
) -> AnyhowResult<Option<RetrivedModelWeight>> {
) -> AnyhowResult<Option<RetrievedModelWeight>> {
let mut connection = mysql_pool.acquire().await?;
get_weights_by_token_with_connection(weight_token, can_see_deleted, &mut connection).await
}
@@ -60,7 +69,7 @@ pub async fn get_weights_by_token_with_connection(
weight_token: &ModelWeightToken,
can_see_deleted: bool,
mysql_connection: &mut PoolConnection<MySql>
) -> AnyhowResult<Option<RetrivedModelWeight>> {
) -> AnyhowResult<Option<RetrievedModelWeight>> {
let maybe_result = if can_see_deleted {
select_include_deleted(weight_token, mysql_connection).await
} else {
@@ -81,7 +90,7 @@ pub async fn get_weights_by_token_with_connection(
// unwrap the result
Ok(
Some(RetrivedModelWeight {
Some(RetrievedModelWeight {
token: record.token,
title: record.title,
weights_type: record.weights_type,
@@ -90,6 +99,9 @@ pub async fn get_weights_by_token_with_connection(
description_markdown: record.description_markdown,
description_rendered_html: record.description_rendered_html,
creator_user_token: record.creator_user_token,
creator_username: record.creator_username,
creator_display_name: record.creator_display_name,
creator_gravatar_hash: record.creator_gravatar_hash,
creator_ip_address: record.creator_ip_address,
creator_set_visibility: record.creator_set_visibility,
maybe_last_update_user_token: record.maybe_last_update_user_token,
@@ -97,9 +109,12 @@ pub async fn get_weights_by_token_with_connection(
original_filename: record.original_filename,
file_size_bytes: record.file_size_bytes,
file_checksum_sha2: record.file_checksum_sha2,
private_bucket_hash: record.private_bucket_hash,
maybe_private_bucket_prefix: record.maybe_private_bucket_prefix,
maybe_private_bucket_extension: record.maybe_private_bucket_extension,
public_bucket_hash: record.public_bucket_hash,
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
maybe_avatar_public_bucket_hash: record.maybe_avatar_public_bucket_hash,
maybe_avatar_public_bucket_prefix: record.maybe_avatar_public_bucket_prefix,
maybe_avatar_public_bucket_extension: record.maybe_avatar_public_bucket_extension,
cached_user_ratings_negative_count: record.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: record.cached_user_ratings_positive_count,
cached_user_ratings_total_count: record.cached_user_ratings_total_count,
@@ -130,7 +145,12 @@ async fn select_include_deleted(
wt.maybe_thumbnail_token,
wt.description_markdown,
wt.description_rendered_html,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash AS creator_gravatar_hash,
wt.creator_ip_address,
wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
@@ -138,9 +158,14 @@ async fn select_include_deleted(
wt.original_filename,
wt.file_size_bytes,
wt.file_checksum_sha2,
wt.private_bucket_hash,
wt.maybe_private_bucket_prefix,
wt.maybe_private_bucket_extension,
wt.public_bucket_hash,
wt.maybe_public_bucket_prefix,
wt.maybe_public_bucket_extension,
avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,
avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,
avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,
wt.cached_user_ratings_negative_count,
wt.cached_user_ratings_positive_count,
wt.cached_user_ratings_total_count,
@@ -152,6 +177,10 @@ async fn select_include_deleted(
wt.user_deleted_at,
wt.mod_deleted_at
FROM model_weights as wt
JOIN users
ON users.token = wt.creator_user_token
LEFT OUTER JOIN media_files as avatar
ON avatar.token = wt.maybe_avatar_media_file_token
WHERE
wt.token = ?
"#,
@@ -178,17 +207,27 @@ async fn select_without_deleted(
wt.maybe_thumbnail_token,
wt.description_markdown,
wt.description_rendered_html,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
wt.creator_ip_address,
wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash AS creator_gravatar_hash,
wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
wt.original_download_url,
wt.original_filename,
wt.file_size_bytes,
wt.file_checksum_sha2,
wt.private_bucket_hash,
wt.maybe_private_bucket_prefix,
wt.maybe_private_bucket_extension,
wt.public_bucket_hash,
wt.maybe_public_bucket_prefix,
wt.maybe_public_bucket_extension,
avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,
avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,
avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,
wt.cached_user_ratings_negative_count,
wt.cached_user_ratings_positive_count,
wt.cached_user_ratings_total_count,
@@ -200,6 +239,10 @@ async fn select_without_deleted(
wt.user_deleted_at,
wt.mod_deleted_at
FROM model_weights as wt
JOIN users
ON users.token = wt.creator_user_token
LEFT OUTER JOIN media_files as avatar
ON avatar.token = wt.maybe_avatar_media_file_token
WHERE
wt.token = ?
AND wt.user_deleted_at IS NULL
@@ -220,7 +263,12 @@ pub struct RawWeight {
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_username: String,
pub creator_display_name: String,
pub creator_gravatar_hash: String,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
@@ -228,9 +276,14 @@ pub struct RawWeight {
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub maybe_avatar_public_bucket_hash: Option<String>,
pub maybe_avatar_public_bucket_prefix: Option<String>,
pub maybe_avatar_public_bucket_extension: Option<String>,
pub cached_user_ratings_negative_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_total_count: u32,
@@ -65,9 +65,9 @@ SELECT
w.title,
vc.has_index_file,
w.private_bucket_hash,
w.maybe_private_bucket_prefix,
w.maybe_private_bucket_extension,
w.public_bucket_hash,
w.maybe_public_bucket_prefix,
w.maybe_public_bucket_extension,
w.created_at,
w.updated_at,
@@ -111,9 +111,9 @@ WHERE w.token = ?
weights_type: model.weights_type,
title: model.title,
has_index_file: i8_to_bool(model.has_index_file),
public_bucket_hash: model.private_bucket_hash,
maybe_public_bucket_prefix: model.maybe_private_bucket_prefix,
maybe_public_bucket_extension: model.maybe_private_bucket_extension,
public_bucket_hash: model.public_bucket_hash,
maybe_public_bucket_prefix: model.maybe_public_bucket_prefix,
maybe_public_bucket_extension: model.maybe_public_bucket_extension,
created_at: model.created_at,
updated_at: model.updated_at,
mod_deleted_at: model.mod_deleted_at,
@@ -129,9 +129,9 @@ struct InternalRecord {
has_index_file: i8,
private_bucket_hash: String,
maybe_private_bucket_prefix: Option<String>,
maybe_private_bucket_extension: Option<String>,
public_bucket_hash: String,
maybe_public_bucket_prefix: Option<String>,
maybe_public_bucket_extension: Option<String>,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
@@ -27,9 +27,9 @@ pub struct WeightsByTokensRecord {
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
@@ -76,9 +76,9 @@ async fn get_raw_weights_by_tokens(
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash as creator_email_gravatar_hash,
mw.private_bucket_hash,
mw.maybe_private_bucket_prefix,
mw.maybe_private_bucket_extension,
mw.public_bucket_hash,
mw.maybe_public_bucket_prefix,
mw.maybe_public_bucket_extension,
mw.cached_user_ratings_negative_count,
mw.cached_user_ratings_positive_count,
mw.cached_user_ratings_total_count,
@@ -109,9 +109,9 @@ async fn get_raw_weights_by_tokens(
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash as creator_email_gravatar_hash,
mw.private_bucket_hash,
mw.maybe_private_bucket_prefix,
mw.maybe_private_bucket_extension,
mw.public_bucket_hash,
mw.maybe_public_bucket_prefix,
mw.maybe_public_bucket_extension,
mw.cached_user_ratings_negative_count,
mw.cached_user_ratings_positive_count,
mw.cached_user_ratings_total_count,
@@ -170,9 +170,9 @@ fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsByTokensRecord>
creator_display_name:dataset.creator_display_name,
creator_email_gravatar_hash:dataset.creator_email_gravatar_hash,
private_bucket_hash: dataset.private_bucket_hash,
maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix,
maybe_private_bucket_extension: dataset.maybe_private_bucket_extension,
public_bucket_hash: dataset.public_bucket_hash,
maybe_public_bucket_prefix: dataset.maybe_public_bucket_prefix,
maybe_public_bucket_extension: dataset.maybe_public_bucket_extension,
cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count,
@@ -207,9 +207,9 @@ fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsByTokensRecord>
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
@@ -247,9 +247,9 @@ impl FromRow<'_, MySqlRow> for RawWeightJoinUser {
creator_username: row.try_get("creator_username")?,
creator_display_name: row.try_get("creator_display_name")?,
creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?,
private_bucket_hash: row.try_get("private_bucket_hash")?,
maybe_private_bucket_prefix: row.try_get("maybe_private_bucket_prefix")?,
maybe_private_bucket_extension: row.try_get("maybe_private_bucket_extension")?,
public_bucket_hash: row.try_get("public_bucket_hash")?,
maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?,
cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?,
cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?,
@@ -0,0 +1,347 @@
use chrono::{DateTime, Utc};
use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
use sqlx::mysql::MySqlRow;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use enums::traits::mysql_from_row::MySqlFromRow;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
pub struct WeightsForUserListPage {
pub records: Vec<WeightsJoinUserRecord>,
pub sort_ascending: bool,
pub current_page: usize,
pub total_page_count: usize,
}
#[derive(Serialize)]
pub struct WeightsJoinUserRecord {
pub token: ModelWeightToken,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub title: String,
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_negative_count: u32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_last_updated_at: DateTime<Utc>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
pub creator_username: String,
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
}
pub struct ListWeightsForUserArgs<'a> {
pub creator_username: &'a str,
pub page_size: usize,
pub page_index: usize,
pub sort_ascending: bool,
pub can_see_deleted: bool,
pub mysql_pool: &'a MySqlPool,
}
pub async fn list_weights_by_creator_username(args: ListWeightsForUserArgs<'_>) -> AnyhowResult<WeightsForUserListPage> {
let count_fields = select_total_count_field();
let mut count_query_builder = query_builder(
args.creator_username,
false,
0,
0,
args.sort_ascending,
count_fields.as_str(),
args.can_see_deleted,
);
let row_count_query = count_query_builder.build_query_scalar::<i64>();
let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?;
/// Now fetch the actual results with all the fields
let result_fields = select_result_fields();
let mut query = query_builder(
args.creator_username,
true,
args.page_index,
args.page_size,
args.sort_ascending,
result_fields.as_str(),
args.can_see_deleted,
);
let query = query.build_query_as::<RawWeightJoinUser>();
let results = query.fetch_all(args.mysql_pool).await?;
let number_of_pages = (row_count_result / args.page_size as i64) as usize;
let weights_records: Vec<WeightsJoinUserRecord> = map_to_weights(results).await;
Ok(WeightsForUserListPage {
records: weights_records,
sort_ascending: args.sort_ascending,
current_page: args.page_index,
total_page_count: number_of_pages,
})
}
fn select_total_count_field() -> String {
r#"
COUNT(mw.id) AS total_count
"#
.to_string()
}
fn select_result_fields() -> String {
r#"
mw.token,
mw.title,
mw.weights_type,
mw.weights_category,
mw.maybe_thumbnail_token,
mw.description_markdown,
mw.description_rendered_html,
u.token as creator_user_token,
u.username as creator_username,
u.display_name as creator_display_name,
u.email_gravatar_hash as creator_email_gravatar_hash,
mw.creator_ip_address,
mw.creator_set_visibility,
mw.maybe_last_update_user_token,
mw.original_download_url,
mw.original_filename,
mw.file_size_bytes,
mw.file_checksum_sha2,
mw.public_bucket_hash,
mw.maybe_public_bucket_prefix,
mw.maybe_public_bucket_extension,
mw.cached_user_ratings_negative_count,
mw.cached_user_ratings_positive_count,
mw.cached_user_ratings_total_count,
mw.maybe_cached_user_ratings_ratio,
mw.cached_user_ratings_last_updated_at,
mw.version,
mw.created_at,
mw.updated_at,
mw.user_deleted_at,
mw.mod_deleted_at
"#.to_string()
}
fn query_builder<'a>(
username: &'a str,
enforce_limits: bool,
page_index: usize,
page_size: usize,
sort_ascending: bool,
select_fields: &'a str,
can_see_deleted: bool,
) -> QueryBuilder<'a, MySql> {
// NB: Query cannot be statically checked by sqlx
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
format!(r#"
SELECT
{select_fields}
FROM model_weights as mw
JOIN users as u
ON u.token = mw.creator_user_token
"#
));
if !can_see_deleted {
query_builder.push(" WHERE mw.user_deleted_at IS NULL AND mw.mod_deleted_at IS NULL ");
}
query_builder.push(" AND u.username = ");
query_builder.push_bind(username);
if sort_ascending {
query_builder.push(" ORDER BY mw.created_at ASC ");
} else {
query_builder.push(" ORDER BY mw.created_at DESC ");
}
if enforce_limits {
let offset = page_index * page_size;
query_builder.push(format!(" LIMIT {page_size} OFFSET {offset} "));
}
query_builder
}
async fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsJoinUserRecord> {
let weights: Vec<WeightsJoinUserRecord> = dataset
.into_iter()
.map(|dataset: RawWeightJoinUser| {
WeightsJoinUserRecord {
token: dataset.token,
title: dataset.title,
weights_type: dataset.weights_type,
weights_category: dataset.weights_category,
maybe_thumbnail_token: dataset.maybe_thumbnail_token,
description_markdown: dataset.description_markdown,
description_rendered_html: dataset.description_rendered_html,
creator_user_token: dataset.creator_user_token,
creator_ip_address: dataset.creator_ip_address,
creator_set_visibility: dataset.creator_set_visibility,
maybe_last_update_user_token: dataset.maybe_last_update_user_token,
original_download_url: dataset.original_download_url,
original_filename: dataset.original_filename,
file_size_bytes: dataset.file_size_bytes,
file_checksum_sha2: dataset.file_checksum_sha2,
public_bucket_hash: dataset.public_bucket_hash,
maybe_public_bucket_prefix: dataset.maybe_public_bucket_prefix,
maybe_public_bucket_extension: dataset.maybe_public_bucket_extension,
cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count,
cached_user_ratings_total_count: dataset.cached_user_ratings_total_count,
maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio,
cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at,
version: dataset.version,
created_at: dataset.created_at,
updated_at: dataset.updated_at,
user_deleted_at: dataset.user_deleted_at,
mod_deleted_at: dataset.mod_deleted_at,
creator_username:dataset.creator_username,
creator_display_name:dataset.creator_display_name,
creator_email_gravatar_hash:dataset.creator_email_gravatar_hash
}
}).collect();
weights
}
pub struct RawWeightJoinUser {
pub token: ModelWeightToken,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub title: String,
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_negative_count: u32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_last_updated_at: DateTime<Utc>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
pub creator_username: String,
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
}
impl FromRow<'_, MySqlRow> for RawWeightJoinUser {
fn from_row(row: &MySqlRow) -> std::result::Result<Self, sqlx::Error> {
Ok(Self {
token: row.try_get("token")?,
weights_type: row.try_get("weights_type")?,
weights_category: row.try_get("weights_category")?,
title: row.try_get("title")?,
maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?,
description_markdown: row.try_get("description_markdown")?,
description_rendered_html: row.try_get("description_rendered_html")?,
creator_user_token: row.try_get("creator_user_token")?,
creator_ip_address: row.try_get("creator_ip_address")?,
creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
maybe_last_update_user_token: row.try_get("maybe_last_update_user_token")?,
original_download_url: row.try_get("original_download_url")?,
original_filename: row.try_get("original_filename")?,
file_size_bytes: row.try_get("file_size_bytes")?,
file_checksum_sha2: row.try_get("file_checksum_sha2")?,
public_bucket_hash: row.try_get("public_bucket_hash")?,
maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?,
cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?,
cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?,
maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?,
cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?,
version: row.try_get("version")?,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
user_deleted_at: row.try_get("user_deleted_at")?,
mod_deleted_at: row.try_get("mod_deleted_at")?,
creator_username: row.try_get("creator_username")?,
creator_display_name: row.try_get("creator_display_name")?,
creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?,
})
}
}
@@ -2,20 +2,18 @@
#![forbid(unused_mut)]
#![forbid(unused_variables)]
use chrono::{ DateTime, Utc };
use chrono::{DateTime, Utc};
use sqlx::MySqlPool;
use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::users::UserToken;
use tokens::tokens::model_weights::ModelWeightToken;
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
};
use tokens::tokens::users::UserToken;
#[derive(Serialize)]
pub struct WeightsPage {
@@ -55,10 +53,9 @@ pub struct WeightJoinUser {
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
@@ -234,9 +231,9 @@ impl ListWeightsQueryBuilder {
original_filename: record.original_filename,
file_size_bytes: record.file_size_bytes,
file_checksum_sha2: record.file_checksum_sha2,
private_bucket_hash: record.private_bucket_hash,
maybe_private_bucket_prefix: record.maybe_private_bucket_prefix,
maybe_private_bucket_extension: record.maybe_private_bucket_extension,
public_bucket_hash: record.public_bucket_hash,
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
cached_user_ratings_total_count: record.cached_user_ratings_total_count,
cached_user_ratings_positive_count: record.cached_user_ratings_positive_count,
cached_user_ratings_negative_count: record.cached_user_ratings_negative_count,
@@ -282,9 +279,9 @@ impl ListWeightsQueryBuilder {
model_weights.original_filename,
model_weights.file_size_bytes,
model_weights.file_checksum_sha2,
model_weights.private_bucket_hash,
model_weights.maybe_private_bucket_prefix,
model_weights.maybe_private_bucket_extension,
model_weights.public_bucket_hash,
model_weights.maybe_public_bucket_prefix,
model_weights.maybe_public_bucket_extension,
model_weights.cached_user_ratings_total_count,
model_weights.cached_user_ratings_positive_count,
model_weights.cached_user_ratings_negative_count,
@@ -442,10 +439,9 @@ struct RawWeightJoinUser {
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
@@ -468,12 +464,13 @@ struct RawWeightJoinUser {
#[cfg(test)]
mod tests {
use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder;
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use crate::queries::model_weights::list::list_weights_query_builder::ListWeightsQueryBuilder;
#[test]
fn predicates_without_scoping() {
let query_builder = ListWeightsQueryBuilder::new();
@@ -1 +1,4 @@
pub mod list_model_weights_for_voice_conversion;
pub mod list_model_weights_for_voice_conversion;
pub mod list_weights_by_tokens;
pub mod list_weights_by_user;
pub mod list_weights_query_builder;
@@ -1,276 +0,0 @@
use chrono::{DateTime, Utc};
use sqlx::{Acquire, MySqlConnection, MySqlPool};
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
#[derive(Serialize)]
pub struct WeightsJoinUserRecord {
pub token: ModelWeightToken,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub title: String,
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_negative_count: u32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_last_updated_at: DateTime<Utc>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
pub creator_username: String,
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
}
pub async fn list_weights_by_creator_username(
mysql_pool: &MySqlPool,
creator_username: &str,
can_see_deleted: bool
) -> AnyhowResult<Vec<WeightsJoinUserRecord>> {
let mut connection = mysql_pool.acquire().await?;
let raw_weights: Vec<RawWeightJoinUser> = get_raw_weights_by_creator_username(&mut connection, creator_username, can_see_deleted).await?;
let weights_records: Vec<WeightsJoinUserRecord> = map_to_weights(raw_weights).await;
Ok(weights_records)
}
async fn get_raw_weights_by_creator_username(
connection: &mut MySqlConnection,
creator_username: &str,
can_see_deleted: bool
) -> AnyhowResult<Vec<RawWeightJoinUser>> {
let connection = connection.acquire().await?;
if can_see_deleted {
let raw_weights: Vec<RawWeightJoinUser> = sqlx::query_as!(
RawWeightJoinUser,
r#"
SELECT
mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
mw.title,
mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,
mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,
mw.maybe_thumbnail_token,
mw.description_markdown,
mw.description_rendered_html,
users.token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash as creator_email_gravatar_hash,
mw.creator_ip_address,
mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
mw.original_download_url,
mw.original_filename,
mw.file_size_bytes,
mw.file_checksum_sha2,
mw.private_bucket_hash,
mw.maybe_private_bucket_prefix,
mw.maybe_private_bucket_extension,
mw.cached_user_ratings_negative_count,
mw.cached_user_ratings_positive_count,
mw.cached_user_ratings_total_count,
mw.maybe_cached_user_ratings_ratio,
mw.cached_user_ratings_last_updated_at,
mw.version,
mw.created_at,
mw.updated_at,
mw.user_deleted_at,
mw.mod_deleted_at
FROM model_weights as mw
JOIN users
ON users.token = mw.creator_user_token
WHERE
users.username = ?
"#,
creator_username).fetch_all(connection)
.await?;
Ok(raw_weights)
} else {
let raw_weights: Vec<RawWeightJoinUser> = sqlx::query_as!(
RawWeightJoinUser,
r#"
SELECT
mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
mw.title,
mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,
mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,
mw.maybe_thumbnail_token,
mw.description_markdown,
mw.description_rendered_html,
users.token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash as creator_email_gravatar_hash,
mw.creator_ip_address,
mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
mw.original_download_url,
mw.original_filename,
mw.file_size_bytes,
mw.file_checksum_sha2,
mw.private_bucket_hash,
mw.maybe_private_bucket_prefix,
mw.maybe_private_bucket_extension,
mw.cached_user_ratings_negative_count,
mw.cached_user_ratings_positive_count,
mw.cached_user_ratings_total_count,
mw.maybe_cached_user_ratings_ratio,
mw.cached_user_ratings_last_updated_at,
mw.version,
mw.created_at,
mw.updated_at,
mw.user_deleted_at,
mw.mod_deleted_at
FROM model_weights as mw
JOIN users
ON users.token = mw.creator_user_token
WHERE
users.username = ?
AND mw.user_deleted_at IS NULL
AND mw.mod_deleted_at IS NULL
"#,
creator_username).fetch_all(connection).await?;
Ok(raw_weights)
}
}
async fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsJoinUserRecord> {
let weights: Vec<WeightsJoinUserRecord> = dataset
.into_iter()
.map(|dataset: RawWeightJoinUser| {
WeightsJoinUserRecord {
token: dataset.token,
title: dataset.title,
weights_type: dataset.weights_type,
weights_category: dataset.weights_category,
maybe_thumbnail_token: dataset.maybe_thumbnail_token,
description_markdown: dataset.description_markdown,
description_rendered_html: dataset.description_rendered_html,
creator_user_token: dataset.creator_user_token,
creator_ip_address: dataset.creator_ip_address,
creator_set_visibility: dataset.creator_set_visibility,
maybe_last_update_user_token: dataset.maybe_last_update_user_token,
original_download_url: dataset.original_download_url,
original_filename: dataset.original_filename,
file_size_bytes: dataset.file_size_bytes,
file_checksum_sha2: dataset.file_checksum_sha2,
private_bucket_hash: dataset.private_bucket_hash,
maybe_private_bucket_prefix: dataset.maybe_private_bucket_prefix,
maybe_private_bucket_extension: dataset.maybe_private_bucket_extension,
cached_user_ratings_negative_count: dataset.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: dataset.cached_user_ratings_positive_count,
cached_user_ratings_total_count: dataset.cached_user_ratings_total_count,
maybe_cached_user_ratings_ratio: dataset.maybe_cached_user_ratings_ratio,
cached_user_ratings_last_updated_at: dataset.cached_user_ratings_last_updated_at,
version: dataset.version,
created_at: dataset.created_at,
updated_at: dataset.updated_at,
user_deleted_at: dataset.user_deleted_at,
mod_deleted_at: dataset.mod_deleted_at,
creator_username:dataset.creator_username,
creator_display_name:dataset.creator_display_name,
creator_email_gravatar_hash:dataset.creator_email_gravatar_hash
}
}).collect();
weights
}
#[derive(Serialize)]
pub struct RawWeightJoinUser {
pub token: ModelWeightToken,
pub weights_type: WeightsType,
pub weights_category: WeightsCategory,
pub title: String,
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub file_checksum_sha2: String,
pub private_bucket_hash: String,
pub maybe_private_bucket_prefix: Option<String>,
pub maybe_private_bucket_extension: Option<String>,
pub cached_user_ratings_total_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_negative_count: u32,
pub maybe_cached_user_ratings_ratio: Option<f32>,
pub cached_user_ratings_last_updated_at: DateTime<Utc>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
pub creator_username: String,
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
}
@@ -32,8 +32,7 @@ pub async fn upsert_model_weight_from_voice_conversion_model(
upsert_model_weights_record(record, &model_weight_token, copied_data, &mut transaction).await?;
upsert_model_weights_extension_record(record, &model_weight_token, &mut transaction).await?;
// TODO: Don't update source record with new token *unless* we're backfilling the same database.
// update_original_record(record, &model_weight_token, &mut transaction).await?;
update_original_record(record, &model_weight_token, &mut transaction).await?;
transaction.commit().await?;
@@ -64,9 +63,8 @@ pub async fn upsert_model_weights_record(
// NB: Not setting a few fields (for now)
// maybe_last_update_user_token - seems like bad design
// TODO(bt): OBVIOUSLY COPY THE BUCKET FILES !
// TODO(bt): file checksum
// TODO(bt): rename maybe_private_bucket_extension to maybe_private_bucket_suffix (!!!)
// TODO(bt): rename maybe_public_bucket_extension to maybe_public_bucket_suffix (!!!)
// TODO(bt): do we need model_weights.ip_address_last_update without audit logs?
// TODO(bt): rename creator_ip_address to ip_address_creation (and add ip_address_last_update)
// TODO(bt): Check model_weights column integer types - signed vs unsigned
@@ -93,9 +91,9 @@ SET
file_size_bytes = ?,
file_checksum_sha2 = ?,
private_bucket_hash = ?,
maybe_private_bucket_prefix = ?,
maybe_private_bucket_extension = ?,
public_bucket_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
cached_user_ratings_total_count = 0,
cached_user_ratings_positive_count = 0,
cached_user_ratings_negative_count = 0,
@@ -126,9 +124,9 @@ ON DUPLICATE KEY UPDATE
original_filename = ?,
file_size_bytes = ?,
file_checksum_sha2 = ?,
private_bucket_hash = ?,
maybe_private_bucket_prefix = ?,
maybe_private_bucket_extension = ?,
public_bucket_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
cached_user_ratings_total_count = 0,
cached_user_ratings_positive_count = 0,
cached_user_ratings_negative_count = 0,
@@ -227,6 +225,18 @@ pub async fn update_original_record(
model_weight_token: &ModelWeightToken,
transaction: &mut Transaction<'_, MySql>
) -> AnyhowResult<()> {
let query = sqlx::query!(
r#"
UPDATE voice_conversion_models
SET
maybe_migration_new_model_weights_token = ?
WHERE token = ?
"#,
model_weight_token,
record.token,
);
let _r = query.execute(&mut **transaction).await?;
Ok(())
}
@@ -1,13 +1,9 @@
pub mod create;
pub mod create_weight;
pub mod delete_weights;
pub mod edit;
pub mod get_weight;
pub mod inference;
pub mod list;
pub mod list_weights_by_tokens;
pub mod list_weights_by_user;
pub mod list_weights_query_builder;
pub mod migration;
pub mod model_weight_info_lite;
pub mod tests;
pub mod update_weight;
@@ -19,9 +19,9 @@ CREATE TABLE model_weights (
original_filename VARCHAR(255) DEFAULT NULL,
file_size_bytes INT(10) NOT NULL DEFAULT 0,
file_checksum_sha2 CHAR(64) NOT NULL,
private_bucket_hash VARCHAR(32) NOT NULL,
maybe_private_bucket_prefix VARCHAR(16) DEFAULT NULL,
maybe_private_bucket_extension VARCHAR(16) DEFAULT NULL,
public_bucket_hash VARCHAR(32) NOT NULL,
maybe_public_bucket_prefix VARCHAR(16) DEFAULT NULL,
maybe_public_bucket_extension VARCHAR(16) DEFAULT NULL,
cached_user_ratings_total_count INT(10) UNSIGNED NOT NULL DEFAULT 0,
cached_user_ratings_positive_count INT(10) UNSIGNED NOT NULL DEFAULT 0,
cached_user_ratings_negative_count INT(10) UNSIGNED NOT NULL DEFAULT 0,
@@ -3,8 +3,8 @@ mod tests {
use anyhow::Ok;
use rand::Rng;
use serial_test::serial;
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use sqlx::MySqlPool;
use tokio;
use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
@@ -17,8 +17,8 @@ mod tests {
use enums::common::visibility::Visibility;
use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
use crate::queries::model_weights::create_weight::create_weight;
use crate::queries::model_weights::create_weight::CreateModelWeightsArgs;
use crate::queries::model_weights::create::create_weight::create_weight;
use crate::queries::model_weights::create::create_weight::CreateModelWeightsArgs;
use crate::queries::model_weights::delete_weights::{
delete_weights_as_mod,
delete_weights_as_user,
@@ -26,8 +26,8 @@ mod tests {
undelete_weights_as_user,
};
use crate::queries::model_weights::get_weight::get_weight_by_token;
use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username;
use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder;
use crate::queries::model_weights::list::list_weights_by_user::{list_weights_by_creator_username, ListWeightsForUserArgs};
use crate::queries::model_weights::list::list_weights_query_builder::ListWeightsQueryBuilder;
use crate::queries::users::user::get_user_token_by_username::get_user_token_by_username;
async fn setup() -> sqlx::Pool<sqlx::MySql> {
@@ -76,9 +76,9 @@ mod tests {
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum1".to_string(),
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: "bucket_hash1".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -112,9 +112,9 @@ mod tests {
assert_eq!(result.original_filename, Some("filename1.txt".to_string()));
assert_eq!(result.file_size_bytes, 1024);
assert_eq!(result.file_checksum_sha2, "checksum1".to_string());
assert_eq!(result.private_bucket_hash, "bucket_hash1".to_string());
assert_eq!(result.maybe_private_bucket_prefix, Some("_fake".to_string()));
assert_eq!(result.maybe_private_bucket_extension, Some("rvc".to_string()));
assert_eq!(result.public_bucket_hash, "bucket_hash1".to_string());
assert_eq!(result.maybe_public_bucket_prefix, Some("_fake".to_string()));
assert_eq!(result.maybe_public_bucket_extension, Some("rvc".to_string()));
assert_eq!(result.cached_user_ratings_total_count, 10);
assert_eq!(result.cached_user_ratings_positive_count, 9);
assert_eq!(result.cached_user_ratings_negative_count, 1);
@@ -151,9 +151,9 @@ mod tests {
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum1".to_string(),
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: "bucket_hash1".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -228,9 +228,9 @@ mod tests {
original_filename: Some("filename1.txt".to_string()),
file_size_bytes: 1024,
file_checksum_sha2: "checksum1".to_string(),
private_bucket_hash: "bucket_hash1".to_string(),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: "bucket_hash1".to_string(),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -290,16 +290,19 @@ mod tests {
&creator_username
).await?;
let weights_by_username = list_weights_by_creator_username(
&pool,
&creator_username,
can_see_deleted
).await?;
for weight in weights_by_username.iter() {
let weights_by_username = list_weights_by_creator_username(ListWeightsForUserArgs {
creator_username: &creator_username,
page_size: 0,
page_index: 0,
sort_ascending: false,
can_see_deleted,
mysql_pool: &pool,
}).await?;
for weight in weights_by_username.records.iter() {
// print weight
println!("weight: {:?}", weight.creator_username);
}
assert_eq!(weights_by_username.len(), 5);
assert_eq!(weights_by_username.records.len(), 5);
Ok(())
}
@@ -340,9 +343,9 @@ mod tests {
original_filename: Some(format!("filename {}.txt", i)),
file_size_bytes: 1024,
file_checksum_sha2: format!("checksum{}", i),
private_bucket_hash: format!("bucket_hash{}", i),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: format!("bucket_hash{}", i),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -583,9 +586,9 @@ mod tests {
original_filename: Some(format!("filename {}.txt", i)),
file_size_bytes: 1024,
file_checksum_sha2: format!("checksum{}", i),
private_bucket_hash: format!("bucket_hash{}", i),
maybe_private_bucket_prefix: Some("_fake".to_string()),
maybe_private_bucket_extension: Some("rvc".to_string()),
public_bucket_hash: format!("bucket_hash{}", i),
maybe_public_bucket_prefix: Some("_fake".to_string()),
maybe_public_bucket_extension: Some("rvc".to_string()),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
@@ -0,0 +1,339 @@
use anyhow::anyhow;
use chrono::{DateTime, Utc};
use log::warn;
use sqlx::{MySql, MySqlPool};
use sqlx::pool::PoolConnection;
use enums::by_table::tts_models::tts_model_type::TtsModelType;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::tts_models::TtsModelToken;
use tokens::tokens::users::UserToken;
use tokens::tokens::vocoder_models::VocoderModelToken;
use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_optional_bool};
/// This is meant to be the entire table
#[derive(Debug)]
pub struct WholeTtsModelRecord {
pub id: i64,
pub token: TtsModelToken,
pub tts_model_type: TtsModelType,
pub text_pipeline_type: Option<String>,
pub has_self_contained_vocoder: bool,
pub has_self_contained_duration_model: bool,
pub has_self_contained_pitch_model: bool,
pub text_preprocessing_algorithm: Option<String>,
pub use_default_mel_multiply_factor: bool,
pub maybe_custom_mel_multiply_factor: Option<f64>,
pub maybe_default_pretrained_vocoder: Option<String>,
pub maybe_custom_vocoder_token: Option<VocoderModelToken>,
pub ietf_language_tag: String,
pub ietf_primary_language_subtag: String,
pub title: String,
pub description_markdown: String,
pub description_rendered_html: String,
pub voice_name: Option<String>,
pub actor_name: Option<String>,
pub characteristic: Option<String>,
pub version_string: Option<String>,
pub is_front_page_featured: bool,
pub is_twitch_featured: bool,
pub maybe_suggested_unique_bot_command: Option<String>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub creator_user_token: UserToken,
pub creator_ip_address_creation: String,
pub creator_ip_address_last_update: String,
pub creator_set_visibility: Visibility,
pub private_bucket_hash: String,
pub private_bucket_object_name: String,
pub private_bucket_object_is_archive: bool,
pub user_ratings_total_count: u32,
pub user_ratings_positive_count: u32,
pub user_ratings_negative_count: u32,
pub is_public_listing_approved: Option<bool>,
pub is_locked_from_user_modification: bool,
pub is_locked_from_use: bool,
pub maybe_mod_comments: Option<String>,
pub maybe_mod_user_token: Option<UserToken>,
pub maybe_migration_new_model_weights_token: Option<ModelWeightToken>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
}
pub async fn list_whole_tts_models_using_cursor(
mysql_pool: &MySqlPool,
page_size: u64,
cursor: u64,
) -> AnyhowResult<Vec<WholeTtsModelRecord>> {
let mut connection = mysql_pool.acquire().await?;
let maybe_models
= list_whole_voice_conversion_models(&mut connection, page_size, cursor)
.await;
let models : Vec<RawRecord> = match maybe_models {
Ok(models) => models,
Err(err) => {
match err {
RowNotFound => {
return Ok(Vec::new());
},
_ => {
warn!("vc model list query error: {:?}", err);
return Err(anyhow!("vc model list query error"));
}
}
}
};
Ok(models.into_iter()
.map(|model| {
WholeTtsModelRecord {
id: model.id,
token: model.token,
tts_model_type: model.tts_model_type,
text_pipeline_type: model.text_pipeline_type,
has_self_contained_vocoder: i8_to_bool(model.has_self_contained_vocoder),
has_self_contained_duration_model: i8_to_bool(model.has_self_contained_duration_model),
has_self_contained_pitch_model: i8_to_bool(model.has_self_contained_pitch_model),
text_preprocessing_algorithm: model.text_preprocessing_algorithm,
use_default_mel_multiply_factor: i8_to_bool(model.use_default_mel_multiply_factor),
maybe_custom_mel_multiply_factor: model.maybe_custom_mel_multiply_factor,
maybe_default_pretrained_vocoder: model.maybe_default_pretrained_vocoder,
title: model.title,
description_markdown: model.description_markdown,
description_rendered_html: model.description_rendered_html,
voice_name: model.voice_name,
actor_name: model.actor_name,
characteristic: model.characteristic,
ietf_language_tag: model.ietf_language_tag,
ietf_primary_language_subtag: model.ietf_primary_language_subtag,
is_front_page_featured: i8_to_bool(model.is_front_page_featured),
is_twitch_featured: i8_to_bool(model.is_twitch_featured),
maybe_suggested_unique_bot_command: model.maybe_suggested_unique_bot_command,
original_download_url: model.original_download_url,
original_filename: model.original_filename,
file_size_bytes: model.file_size_bytes,
creator_user_token: model.creator_user_token,
creator_ip_address_creation: model.creator_ip_address_creation,
creator_ip_address_last_update: model.creator_ip_address_last_update,
creator_set_visibility: model.creator_set_visibility,
private_bucket_hash: model.private_bucket_hash,
private_bucket_object_name: model.private_bucket_object_name,
private_bucket_object_is_archive: i8_to_bool(model.private_bucket_object_is_archive),
user_ratings_total_count: model.user_ratings_total_count,
user_ratings_positive_count: model.user_ratings_positive_count,
user_ratings_negative_count: model.user_ratings_negative_count,
is_public_listing_approved: nullable_i8_to_optional_bool(model.is_public_listing_approved),
is_locked_from_user_modification: i8_to_bool(model.is_locked_from_user_modification),
is_locked_from_use: i8_to_bool(model.is_locked_from_use),
maybe_mod_comments: model.maybe_mod_comments,
maybe_mod_user_token: model.maybe_mod_user_token,
maybe_migration_new_model_weights_token: model.maybe_migration_new_model_weights_token,
version: model.version,
created_at: model.created_at,
updated_at: model.updated_at,
user_deleted_at: model.user_deleted_at,
mod_deleted_at: model.mod_deleted_at,
maybe_custom_vocoder_token: model.maybe_custom_vocoder_token,
version_string: model.version_string
}
})
.collect::<Vec<WholeTtsModelRecord>>())
}
async fn list_whole_voice_conversion_models(
mysql_connection: &mut PoolConnection<MySql>,
page_size: u64,
cursor: u64,
) -> AnyhowResult<Vec<RawRecord>> {
Ok(sqlx::query_as!(
RawRecord,
r#"
SELECT
id,
token as `token: tokens::tokens::tts_models::TtsModelToken`,
tts_model_type as `tts_model_type: enums::by_table::tts_models::tts_model_type::TtsModelType`,
text_pipeline_type,
has_self_contained_vocoder,
has_self_contained_duration_model,
has_self_contained_pitch_model,
text_preprocessing_algorithm,
use_default_mel_multiply_factor,
maybe_custom_mel_multiply_factor,
maybe_default_pretrained_vocoder,
maybe_custom_vocoder_token as `maybe_custom_vocoder_token: tokens::tokens::vocoder_models::VocoderModelToken`,
ietf_language_tag,
ietf_primary_language_subtag,
title,
description_markdown,
description_rendered_html,
voice_name,
actor_name,
characteristic,
version_string,
is_front_page_featured,
is_twitch_featured,
maybe_suggested_unique_bot_command,
original_download_url,
original_filename,
file_size_bytes,
creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
creator_ip_address_creation,
creator_ip_address_last_update,
creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
private_bucket_hash,
private_bucket_object_name,
private_bucket_object_is_archive,
user_ratings_total_count,
user_ratings_positive_count,
user_ratings_negative_count,
is_public_listing_approved,
is_locked_from_user_modification,
is_locked_from_use,
maybe_mod_comments,
maybe_mod_user_token as `maybe_mod_user_token: tokens::tokens::users::UserToken`,
maybe_migration_new_model_weights_token as `maybe_migration_new_model_weights_token: tokens::tokens::model_weights::ModelWeightToken`,
version,
created_at,
updated_at,
user_deleted_at,
mod_deleted_at
FROM tts_models
WHERE
id > ?
ORDER BY id ASC
LIMIT ?
"#,
cursor,
page_size
)
.fetch_all(&mut **mysql_connection)
.await?)
}
struct RawRecord {
pub id: i64,
pub token: TtsModelToken,
pub tts_model_type: TtsModelType,
pub text_pipeline_type: Option<String>,
pub has_self_contained_vocoder: i8, // bool
pub has_self_contained_duration_model: i8, // bool
pub has_self_contained_pitch_model: i8, // bool
pub text_preprocessing_algorithm: Option<String>,
pub use_default_mel_multiply_factor: i8, // bool
pub maybe_custom_mel_multiply_factor: Option<f64>,
pub maybe_default_pretrained_vocoder: Option<String>,
pub maybe_custom_vocoder_token: Option<VocoderModelToken>,
pub ietf_language_tag: String,
pub ietf_primary_language_subtag: String,
pub title: String,
pub description_markdown: String,
pub description_rendered_html: String,
pub voice_name: Option<String>,
pub actor_name: Option<String>,
pub characteristic: Option<String>,
pub version_string: Option<String>,
pub is_front_page_featured: i8, // bool
pub is_twitch_featured: i8, // bool
pub maybe_suggested_unique_bot_command: Option<String>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
pub file_size_bytes: i32,
pub creator_user_token: UserToken,
pub creator_ip_address_creation: String,
pub creator_ip_address_last_update: String,
pub creator_set_visibility: Visibility,
pub private_bucket_hash: String,
pub private_bucket_object_name: String,
pub private_bucket_object_is_archive: i8, // bool
pub user_ratings_total_count: u32,
pub user_ratings_positive_count: u32,
pub user_ratings_negative_count: u32,
pub is_public_listing_approved: Option<i8>, // bool
pub is_locked_from_user_modification: i8, // bool
pub is_locked_from_use: i8, // bool
pub maybe_mod_comments: Option<String>,
pub maybe_mod_user_token: Option<UserToken>,
pub maybe_migration_new_model_weights_token: Option<ModelWeightToken>,
pub version: i32,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub user_deleted_at: Option<DateTime<Utc>>,
pub mod_deleted_at: Option<DateTime<Utc>>,
}
@@ -0,0 +1 @@
pub mod list_whole_tts_models_using_cursor;
@@ -9,3 +9,4 @@ pub mod get_tts_model_for_inference_improved;
pub mod insert_tts_model_from_download_job;
pub mod list_all_tts_model_tokens;
pub mod list_tts_models;
pub mod migration;
@@ -6,7 +6,7 @@ use log::{info, warn};
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::tts_models::tts_model_type::TtsModelType;
use filesys::file_exists::file_exists;
use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelType};
use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelError, VcModelType};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord};
@@ -163,7 +163,11 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie
let maybe_vc_model = query_vc_model_for_migration(token,
&job_dependencies.db.mysql_pool)
.await
.map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?;
.map_err(|err|
match err {
VcModelError::ModelDeleted => ProcessSingleJobError::ModelDeleted,
_ => ProcessSingleJobError::Other(anyhow!("database error: {:?}", err))
})?;
match maybe_vc_model {
None => MaybeInferenceModel::None,
@@ -13,7 +13,7 @@ use filesys::safe_delete_temp_file::safe_delete_temp_file;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs};
use tokens::tokens::users::UserToken;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -13,7 +13,7 @@ use hashing::sha256::sha256_hash_string::sha256_hash_string;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::column_types::vocoder_type::VocoderType;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs};
use mysql_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result;
use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType;
@@ -13,8 +13,8 @@ use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::file_size::file_size;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mysql_queries::queries::media_files::insert_media_file_from_zero_shot_tts::insert_media_file_from_zero_shot;
use mysql_queries::queries::media_files::insert_media_file_from_zero_shot_tts::InsertArgs;
use mysql_queries::queries::media_files::create::insert_media_file_from_zero_shot_tts::insert_media_file_from_zero_shot;
use mysql_queries::queries::media_files::create::insert_media_file_from_zero_shot_tts::InsertArgs;
use mysql_queries::queries::voice_designer::voices::get_voice::{get_voice_by_token, ZsVoice};
use crate::job::job_loop::job_success_result::JobSuccessResult;
@@ -19,7 +19,7 @@ use migration::voice_conversion::query_vc_model_for_migration::VcModel;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_files::create::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::model_weights::ModelWeightToken;
@@ -18,7 +18,7 @@ use migration::voice_conversion::query_vc_model_for_migration::VcModel;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_files::create::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::model_weights::ModelWeightToken;
@@ -7,7 +7,7 @@ use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::model_weights::get_weight::RetrivedModelWeight;
use mysql_queries::queries::model_weights::get_weight::RetrievedModelWeight;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
@@ -18,7 +18,7 @@ pub struct ModelFile {
}
pub async fn download_model_file(
model_record: &Option<RetrivedModelWeight>,
model_record: &Option<RetrievedModelWeight>,
public_bucket_client: &BucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
@@ -32,12 +32,12 @@ pub async fn download_model_file(
Some(model_record) => {
let model_bucket_path =
MediaFileBucketPath::from_object_hash(
&model_record.private_bucket_hash,
model_record.maybe_private_bucket_prefix.as_deref(),
model_record.maybe_private_bucket_extension.as_deref(),
&model_record.public_bucket_hash,
model_record.maybe_public_bucket_prefix.as_deref(),
model_record.maybe_public_bucket_extension.as_deref(),
);
let base_name = &model_record.private_bucket_hash;
let extension = model_record.maybe_private_bucket_extension.as_deref().unwrap_or("bin");
let base_name = &model_record.public_bucket_hash;
let extension = model_record.maybe_public_bucket_extension.as_deref().unwrap_or("bin");
model_filename = format!("{base_name}.{extension}");
bucket_object_path = model_bucket_path.to_full_object_pathbuf();
}
@@ -4,8 +4,8 @@ use std::time::Instant;
use anyhow::{anyhow, Error};
use log::{error, info, warn};
use serde::{Deserialize, Serialize};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::file_size::file_size;
@@ -15,7 +15,7 @@ use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs};
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -28,7 +28,6 @@ use sqlx::mysql::MySqlPoolOptions;
use sqlx::MySqlPool;
use tempdir::TempDir;
use buckets::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
use cloud_storage::bucket_client::BucketClient;
use config::common_env::CommonEnv;
use config::is_bad_video_download_url::is_bad_video_download_url;
@@ -474,9 +473,11 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord)
info!("File hash: {}", private_bucket_hash);
// Full path to video/image
let full_object_path = hash_to_bucket_path_string(
&private_bucket_hash,
Some(&downloader.bucket_root_w2l_template_uploads))?;
//let full_object_path = hash_to_bucket_path_string(
// &private_bucket_hash,
// Some(&downloader.bucket_root_w2l_template_uploads))?;
let full_object_path = "THIS_IS_BROKEN_BECAUSE_W2L_IS_DEAD";
// ==================== GENERATE VIDEO PREVIEWS ==================== //
@@ -1,4 +1,3 @@
use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*;
use crate::http_server::endpoints::weights::list_available_weights::*;
@@ -8,26 +7,50 @@ use crate::http_server::endpoints::weights::get_weight::*;
use crate::http_server::endpoints::weights::update_weight::*;
use crate::http_server::endpoints::weights::list_weights_by_user::*;
use crate::http_server::common_responses::user_details_lite::{DefaultAvatarInfo, UserDetailsLight};
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
};
use crate::http_server::common_responses::media_file_social_meta_lite::MediaFileSocialMetaLight;
use crate::http_server::common_responses::pagination_cursors::PaginationCursors;
use crate::http_server::common_responses::pagination_page::PaginationPage;
use crate::http_server::endpoints::media_files::list_featured_media_files::*;
use crate::http_server::endpoints::media_files::list_media_files::*;
use crate::http_server::endpoints::media_files::list_media_files_for_user::*;
use enums::by_table::model_weights::{weights_types::WeightsType, weights_category::WeightsCategory};
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::*;
use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::*;
use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::*;
use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::*;
use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::*;
use crate::http_server::endpoints::user_ratings::get_user_rating_handler::*;
use crate::http_server::endpoints::user_ratings::set_user_rating_handler::*;
use tokens::tokens::generic_inference_jobs::*;
use tokens::tokens::model_weights::*;
use tokens::tokens::users::*;
use tokens::tokens::media_files::*;
use tokens::tokens::user_bookmarks::*;
use enums::common::visibility::Visibility;
use chrono::{DateTime, Utc};
use utoipa::OpenApi;
#[derive(OpenApi)]
#[openapi(paths(crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request,
crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler,
crate::http_server::endpoints::weights::delete_weight::delete_weight_handler,
crate::http_server::endpoints::weights::update_weight::update_weight_handler,
crate::http_server::endpoints::weights::get_weight::get_weight_handler,
crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler
crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler,
crate::http_server::endpoints::media_files::list_featured_media_files::list_featured_media_files_handler,
crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler,
crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler,
crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler,
crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler,
crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler,
crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler,
),components(schemas(
// Tokens
UserToken,Visibility,
@@ -40,10 +63,22 @@ use utoipa::OpenApi;
GetWeightPathInfo,GetWeightResponse,GetWeightError,
UpdateWeightRequest,UpdateWeightRequest,UpdateWeightPathInfo,UpdateWeightRequest,UpdateWeightError,
DeleteWeightPathInfo,DeleteWeightRequest,DeleteWeightError,DeleteWeightRequest,DeleteWeightError,
ListWeightError,ModelWeightForList,ListWeightError,
ListWeightError,ModelWeightForList,ListWeightError,
ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList,
ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight,
UserDetailsLight,DefaultAvatarInfo
UserDetailsLight,DefaultAvatarInfo,
PaginationCursors,
PaginationPage,
MediaFileOriginModelType,MediaFileOriginProductCategory,MediaFileOriginCategory,
ListFeaturedMediaFilesSuccessResponse,MediaFile,MediaFileToken,MediaFileType, ListFeaturedMediaFilesError,
ListMediaFilesSuccessResponse, ListMediaFilesError, MediaFileListItem, ListMediaFilesQueryParams,
MediaFileSocialMetaLight,
ListMediaFilesForUserSuccessResponse,ListMediaFilesForUserQueryParams,ListMediaFilesForUserError,ListMediaFilesForUserPathInfo,
MediaFileForUserListItem, MediaFileForUserListItem,
UserBookmarkToken,UserBookmarkDetailsForUserList,UserBookmarkEntityType,
CreateUserBookmarkRequest,CreateUserBookmarkError,CreateUserBookmarkSuccessResponse,
DeleteUserBookmarkPathInfo,DeleteUserBookmarkError,DeleteUserBookmarkRequest,
ListUserBookmarksPathInfo,ListUserBookmarksForUserError,ListUserBookmarksForUserSuccessResponse,UserBookmarkListItem,
ListUserBookmarksForEntityPathInfo,ListUserBookmarksForEntityError,ListUserBookmarksForEntitySuccessResponse,UserBookmarkForEntityListItem,
)))]
pub struct ApiDoc;
@@ -0,0 +1,22 @@
use utoipa::ToSchema;
use tokens::tokens::media_files::MediaFileToken;
/// Fields useful for enriching media file listings
#[derive(Clone, Serialize, ToSchema)]
pub struct MediaFileSocialMetaLight {
pub favorites_count: u64,
pub comments_count: u64,
}
impl MediaFileSocialMetaLight {
pub fn from_db_fields(
favorites_count: u64,
comments_count: u64,
) -> Self {
Self {
favorites_count,
comments_count,
}
}
}
@@ -1,3 +1,4 @@
pub mod pagination_cursors;
pub mod pagination_page;
pub mod user_details_lite;
pub mod user_details_lite;
pub mod media_file_social_meta_lite;
@@ -1,10 +1,12 @@
use utoipa::ToSchema;
/// Pagination by cursors
/// These types of pagination are for "infinite scrolling", which do not reveal a number of pages.
/// This is good so that investors and competitors cannot reveal how many database records we have.
/// This is typically used for "discovery" type pages, not user profiles.
///
/// See `PaginationPage` for the other type of pagination.
#[derive(Serialize)]
#[derive(Serialize, ToSchema)]
pub struct PaginationCursors {
/// The "next" cursor.
@@ -1,3 +1,5 @@
use utoipa::ToSchema;
/// Pagination by page id
/// This type of pagination is by "page id" and "page count".
/// This should never be used to walk the entire public database as competitors and investors
@@ -5,7 +7,7 @@
/// This is best used for user profiles and scoped down results.
///
/// See `PaginationCursors` for the other type of pagination.
#[derive(Serialize)]
#[derive(Serialize, ToSchema)]
pub struct PaginationPage {
/// The current page number
pub current: usize,
@@ -7,20 +7,11 @@ use chrono::{DateTime, Utc};
use log::{debug, error, warn};
use r2d2_redis::redis::Commands;
use utoipa::ToSchema;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use mysql_queries::queries::media_files::list_media_files_by_tokens::list_media_files_by_tokens;
use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_type::MediaFileType;
use mysql_queries::queries::media_files::list::list_media_files_by_tokens::list_media_files_by_tokens;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
use crate::server_state::ServerState;
@@ -3,25 +3,28 @@ use std::sync::Arc;
use actix_web::{HttpMessage, HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Path, Query};
use actix_web::web::Query;
use chrono::{DateTime, Utc};
use log::{info, warn};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use mysql_queries::queries::media_files::list_media_files::{list_media_files, ListMediaFilesArgs, ViewAs};
use mysql_queries::queries::media_files::list::list_media_files::{list_media_files, ListMediaFilesArgs, ViewAs};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::common_responses::media_file_social_meta_lite::MediaFileSocialMetaLight;
use crate::http_server::common_responses::pagination_cursors::PaginationCursors;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
#[derive(Deserialize)]
pub struct QueryParams {
#[derive(Deserialize, ToSchema, IntoParams)]
pub struct ListMediaFilesQueryParams {
pub sort_ascending: Option<bool>,
pub per_page: Option<usize>,
pub cursor: Option<String>,
@@ -29,14 +32,14 @@ pub struct QueryParams {
pub filter_media_type: Option<MediaFileType>,
}
#[derive(Serialize)]
pub struct SuccessResponse {
#[derive(Serialize, ToSchema)]
pub struct ListMediaFilesSuccessResponse {
pub success: bool,
pub results: Vec<MediaFileListItem>,
pub pagination: PaginationCursors,
}
#[derive(Serialize)]
#[derive(Serialize, ToSchema)]
pub struct MediaFileListItem {
pub token: MediaFileToken,
@@ -46,11 +49,12 @@ pub struct MediaFileListItem {
pub maybe_origin_model_token: Option<String>,
pub media_type: MediaFileType,
pub public_bucket_directory_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
/// URL to the media file.
pub public_bucket_path: String,
pub maybe_creator: Option<UserDetailsLight>,
pub maybe_social_meta: Option<MediaFileSocialMetaLight>,
pub creator_set_visibility: Visibility,
@@ -58,38 +62,49 @@ pub struct MediaFileListItem {
pub updated_at: DateTime<Utc>,
}
#[derive(Debug)]
pub enum ErrorResponse {
#[derive(Debug, ToSchema)]
pub enum ListMediaFilesError {
ServerError,
}
impl ResponseError for ErrorResponse {
impl ResponseError for ListMediaFilesError {
fn status_code(&self) -> StatusCode {
match *self {
ErrorResponse::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
ListMediaFilesError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
ErrorResponse::ServerError => "server error".to_string(),
ListMediaFilesError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
impl std::fmt::Display for ErrorResponse {
impl std::fmt::Display for ListMediaFilesError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
#[utoipa::path(
get,
path = "/v1/media_files/list",
params(
ListMediaFilesQueryParams,
),
responses(
(status = 200, description = "List Featured Media Files", body = ListMediaFilesSuccessResponse),
(status = 500, description = "Server error", body = ListMediaFilesError),
),
)]
pub async fn list_media_files_handler(
http_request: HttpRequest,
query: Query<QueryParams>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ErrorResponse>
http_request: HttpRequest,
query: Query<ListMediaFilesQueryParams>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ListMediaFilesError>
{
let maybe_user_session = server_state
.session_checker
@@ -97,7 +112,7 @@ pub async fn list_media_files_handler(
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
ErrorResponse::ServerError
ListMediaFilesError::ServerError
})?;
let mut is_mod = false;
@@ -119,7 +134,7 @@ pub async fn list_media_files_handler(
let cursor = server_state.sort_key_crypto.decrypt_id(cursor)
.map_err(|e| {
warn!("crypto error: {:?}", e);
ErrorResponse::ServerError
ListMediaFilesError::ServerError
})?;
Some(cursor as usize)
} else {
@@ -146,7 +161,7 @@ pub async fn list_media_files_handler(
Ok(results) => results,
Err(e) => {
warn!("Query error: {:?}", e);
return Err(ErrorResponse::ServerError);
return Err(ListMediaFilesError::ServerError);
}
};
@@ -154,7 +169,7 @@ pub async fn list_media_files_handler(
let cursor = server_state.sort_key_crypto.encrypt_id(id as u64)
.map_err(|e| {
warn!("crypto error: {:?}", e);
ErrorResponse::ServerError
ListMediaFilesError::ServerError
})?;
Some(cursor)
} else {
@@ -165,7 +180,7 @@ pub async fn list_media_files_handler(
let cursor = server_state.sort_key_crypto.encrypt_id(id as u64)
.map_err(|e| {
warn!("crypto error: {:?}", e);
ErrorResponse::ServerError
ListMediaFilesError::ServerError
})?;
Some(cursor)
} else {
@@ -174,28 +189,35 @@ pub async fn list_media_files_handler(
let results = results_page.records.into_iter()
.map(|record| MediaFileListItem {
token: record.token,
token: record.token.clone(),
origin_category: record.origin_category,
origin_product_category: record.origin_product_category,
maybe_origin_model_type: record.maybe_origin_model_type,
maybe_origin_model_token: record.maybe_origin_model_token,
media_type: record.media_type,
public_bucket_directory_hash: record.public_bucket_directory_hash,
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
public_bucket_path: MediaFileBucketPath::from_object_hash(
&record.public_bucket_directory_hash,
record.maybe_public_bucket_prefix.as_deref(),
record.maybe_public_bucket_extension.as_deref())
.get_full_object_path_str()
.to_string(),
maybe_creator: UserDetailsLight::from_optional_db_fields_owned(
record.maybe_creator_user_token,
record.maybe_creator_username,
record.maybe_creator_display_name,
record.maybe_creator_gravatar_hash,
),
maybe_social_meta: Option::from(MediaFileSocialMetaLight::from_db_fields(
record.favorite_count,
record.comment_count,
)),
creator_set_visibility: record.creator_set_visibility,
created_at: record.created_at,
updated_at: record.updated_at,
})
.collect::<Vec<_>>();
let response = SuccessResponse {
let response = ListMediaFilesSuccessResponse {
success: true,
results,
pagination: PaginationCursors {
@@ -206,7 +228,7 @@ pub async fn list_media_files_handler(
};
let body = serde_json::to_string(&response)
.map_err(|e| ErrorResponse::ServerError)?;
.map_err(|e| ListMediaFilesError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -5,45 +5,44 @@ use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Path, Query};
use chrono::{DateTime, Utc};
use log::{info, warn};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs};
use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::common_responses::pagination_cursors::PaginationCursors;
use crate::http_server::common_responses::pagination_page::PaginationPage;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
#[derive(Deserialize)]
pub struct PathInfo {
#[derive(Deserialize, ToSchema)]
pub struct ListMediaFilesForUserPathInfo {
username: String,
}
#[derive(Deserialize)]
pub struct QueryParams {
#[derive(Deserialize, ToSchema, IntoParams)]
pub struct ListMediaFilesForUserQueryParams {
pub sort_ascending: Option<bool>,
pub page_size: Option<usize>,
pub page_index: Option<usize>,
pub cursor: Option<String>,
pub cursor_is_reversed: Option<bool>,
pub filter_media_type: Option<MediaFileType>,
}
#[derive(Serialize)]
pub struct SuccessResponse {
#[derive(Serialize, ToSchema)]
pub struct ListMediaFilesForUserSuccessResponse {
pub success: bool,
pub results: Vec<MediaFileListItem>,
pub results: Vec<MediaFileForUserListItem>,
pub pagination: PaginationPage,
}
#[derive(Serialize)]
pub struct MediaFileListItem {
#[derive(Serialize, ToSchema)]
pub struct MediaFileForUserListItem {
pub token: MediaFileToken,
pub origin_category: MediaFileOriginCategory,
@@ -52,9 +51,9 @@ pub struct MediaFileListItem {
pub maybe_origin_model_token: Option<String>,
pub media_type: MediaFileType,
pub public_bucket_directory_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
/// URL to the media file.
pub public_bucket_path: String,
pub creator_set_visibility: Visibility,
@@ -62,39 +61,50 @@ pub struct MediaFileListItem {
pub updated_at: DateTime<Utc>,
}
#[derive(Debug)]
pub enum ErrorResponse {
#[derive(Debug, ToSchema)]
pub enum ListMediaFilesForUserError {
ServerError,
}
impl ResponseError for ErrorResponse {
impl ResponseError for ListMediaFilesForUserError {
fn status_code(&self) -> StatusCode {
match *self {
ErrorResponse::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
ListMediaFilesForUserError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
ErrorResponse::ServerError => "server error".to_string(),
ListMediaFilesForUserError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
impl std::fmt::Display for ErrorResponse {
impl std::fmt::Display for ListMediaFilesForUserError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
#[utoipa::path(
get,
path = "/v1/media_files/list/user/{username}",
params(
ListMediaFilesForUserQueryParams,
),
responses(
(status = 200, description = "List Featured Media Files", body = ListMediaFilesForUserSuccessResponse),
(status = 500, description = "Server error", body = ListMediaFilesForUserError),
),
)]
pub async fn list_media_files_for_user_handler(
http_request: HttpRequest,
path: Path<PathInfo>,
query: Query<QueryParams>,
path: Path<ListMediaFilesForUserPathInfo>,
query: Query<ListMediaFilesForUserQueryParams>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ErrorResponse>
) -> Result<HttpResponse, ListMediaFilesForUserError>
{
let maybe_user_session = server_state
.session_checker
@@ -102,7 +112,7 @@ pub async fn list_media_files_for_user_handler(
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
ErrorResponse::ServerError
ListMediaFilesForUserError::ServerError
})?;
let mut is_author = false;
@@ -118,10 +128,7 @@ pub async fn list_media_files_for_user_handler(
// TODO(bt,2023-12-04): Enforce real maximums and defaults
let limit = query.page_size.unwrap_or(25);
let sort_ascending = query.sort_ascending.unwrap_or(false);
let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false);
let page_size = query.page_size.unwrap_or_else(|| 25);
let page_index = query.page_index.unwrap_or_else(|| 0);
@@ -139,7 +146,7 @@ pub async fn list_media_files_for_user_handler(
maybe_filter_media_type: query.filter_media_type,
page_size,
page_index,
cursor_is_reversed,
sort_ascending,
view_as,
mysql_pool: &server_state.mysql_pool,
}).await;
@@ -149,29 +156,32 @@ pub async fn list_media_files_for_user_handler(
Ok(results) => results,
Err(e) => {
warn!("Query error: {:?}", e);
return Err(ErrorResponse::ServerError);
return Err(ListMediaFilesForUserError::ServerError);
}
};
let results = results_page.records.into_iter()
.map(|record| MediaFileListItem {
.map(|record| MediaFileForUserListItem {
token: record.token,
origin_category: record.origin_category,
origin_product_category: record.origin_product_category,
maybe_origin_model_type: record.maybe_origin_model_type,
maybe_origin_model_token: record.maybe_origin_model_token,
media_type: record.media_type,
public_bucket_directory_hash: record.public_bucket_directory_hash,
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
public_bucket_path: MediaFileBucketPath::from_object_hash(
&record.public_bucket_directory_hash,
record.maybe_public_bucket_prefix.as_deref(),
record.maybe_public_bucket_extension.as_deref())
.get_full_object_path_str()
.to_string(),
creator_set_visibility: record.creator_set_visibility,
created_at: record.created_at,
updated_at: record.updated_at,
})
.collect::<Vec<_>>();
let response = SuccessResponse {
let response = ListMediaFilesForUserSuccessResponse {
success: true,
results,
pagination: PaginationPage{
@@ -181,7 +191,7 @@ pub async fn list_media_files_for_user_handler(
};
let body = serde_json::to_string(&response)
.map_err(|e| ErrorResponse::ServerError)?;
.map_err(|e| ListMediaFilesForUserError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -5,3 +5,5 @@ pub mod list_featured_media_files;
pub mod list_media_files;
pub mod list_media_files_for_user;
pub mod update_media_file;
pub mod upload;
pub mod upload_media_file;
@@ -0,0 +1,91 @@
use actix_multipart::Multipart;
use actix_web::web::BytesMut;
use futures::TryStreamExt;
use log::warn;
use errors::AnyhowResult;
use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken;
use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text};
pub struct MediaFileUploadData {
pub uuid_idempotency_token: Option<String>,
pub file_name: Option<String>,
pub file_bytes: Option<BytesMut>,
pub media_source: MediaFileUploadSource,
}
/// Where the frontend tells us the file came from.
#[derive(Eq, PartialEq, Clone, Copy)]
pub enum MediaFileUploadSource {
Unknown,
/// Eg. from the user's filesystem
UserFile,
/// Eg. the web audio API in Javascript.
UserDeviceApi,
}
/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads.
pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult<MediaFileUploadData> {
let mut uuid_idempotency_token = None;
let mut file_bytes = None;
let mut file_name = None;
let mut media_source = None;
while let Ok(Some(mut field)) = multipart_payload.try_next().await {
let mut field_name = None;
let mut field_filename = None;
if let content_disposition = field.content_disposition() {
field_name = content_disposition.get_name()
.map(|s| s.to_string());
field_filename = content_disposition.get_filename() // NB: Only used for the file bytes.
.map(|s| s.to_string());
}
match field_name.as_deref() {
Some("uuid_idempotency_token") => {
uuid_idempotency_token = read_multipart_field_as_text(&mut field).await
.map_err(|e| {
warn!("Error reading idempotency token: {:}", &e);
e
})?;
},
Some("file") => {
file_name = field_filename.clone();
file_bytes = checked_read_multipart_bytes(&mut field).await
.map_err(|e| {
warn!("Error reading audio upload: {:}", &e);
e
})?;
},
Some("source") => {
media_source = read_multipart_field_as_text(&mut field).await
.map_err(|e| {
warn!("Error reading source: {:}", &e);
e
})?;
},
_ => continue,
}
}
let media_source = match media_source.as_deref() {
Some("device") => MediaFileUploadSource::UserDeviceApi,
Some("file") => MediaFileUploadSource::UserFile,
_ => {
if file_name.as_deref() == Some("blob") {
MediaFileUploadSource::UserDeviceApi
} else {
MediaFileUploadSource::Unknown
}
},
};
Ok(MediaFileUploadData {
uuid_idempotency_token,
file_name,
file_bytes,
media_source,
})
}
@@ -0,0 +1,3 @@
pub mod drain_multipart_request;
pub mod process_upload_media_file;
pub mod upload_error;
@@ -0,0 +1,344 @@
use std::collections::HashSet;
use std::sync::Arc;
use actix_multipart::Multipart;
use actix_web::{HttpRequest, web};
use log::{error, info, warn};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mysql_queries::queries::media_files::create::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upload::drain_multipart_request::{drain_multipart_request, MediaFileUploadSource};
use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
use crate::server_state::ServerState;
use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format;
// TODO(bt,2023-12-20): THIS CODE NEEDS CLEANUP. This has been cargo culted three+ times.
// It's ridiculous and complicated.
pub enum SuccessCase {
MediaAlreadyUploaded {
existing_media_file_token: MediaFileToken,
},
MediaSuccessfullyUploaded {
media_file_token: MediaFileToken,
}
}
impl SuccessCase {
pub fn to_media_token(self) -> MediaFileToken {
match self {
SuccessCase::MediaAlreadyUploaded { existing_media_file_token } => existing_media_file_token,
SuccessCase::MediaSuccessfullyUploaded { media_file_token } => media_file_token,
}
}
}
pub async fn process_upload_media_file(
http_request: &HttpRequest,
server_state: &web::Data<Arc<ServerState>>,
mut multipart_payload: Multipart,
allowed_mimetypes: &HashSet<&'static str>,
) -> Result<SuccessCase, MediaFileUploadError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await
.map_err(|err| {
error!("MySql pool error: {:?}", err);
MediaFileUploadError::ServerError
})?;
// ==================== READ SESSION ==================== //
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(http_request, &mut mysql_connection)
.await
.map_err(|e| {
error!("Session checker error: {:?}", e);
MediaFileUploadError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.is_banned {
return Err(MediaFileUploadError::NotAuthorized);
}
}
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.file_upload_logged_out,
Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in,
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
return Err(MediaFileUploadError::RateLimited);
}
// ==================== READ MULTIPART REQUEST ==================== //
let upload_media_request = drain_multipart_request(multipart_payload)
.await
.map_err(|e| {
// TODO: Error handling could be nicer.
MediaFileUploadError::BadInput("bad request".to_string())
})?;
let uuid_idempotency_token = upload_media_request.uuid_idempotency_token
.ok_or(MediaFileUploadError::BadInput("no uuid".to_string()))?;
// TODO(bt,2023-12-20): Generic table for this.
//let maybe_existing_upload =
// get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection)
// .await;
//match maybe_existing_upload {
// Err(err) => {
// error!("Error checking for previous upload: {:?}", err);
// return Err(MediaFileUploadError::ServerError);
// }
// Ok(Some(upload)) => {
// // File already uploaded and frontend didn't protect us.
// return Ok(SuccessCase::MediaAlreadyUploaded {
// existing_media_file_token: upload.token,
// });
// }
// Ok(None) => {
// // Proceed.
// }
//}
if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) {
return Err(MediaFileUploadError::BadInput(reason));
}
let creator_set_visibility = maybe_user_session
.as_ref()
.map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control.
.unwrap_or(Visibility::default());
let ip_address = get_request_ip(&http_request);
let maybe_user_token = maybe_user_session
.map(|session| session.get_strongly_typed_user_token());
let maybe_file_size_bytes = upload_media_request.file_bytes
.as_ref()
.map(|bytes| bytes.len());
info!("Upload maybe filesize: {:?}", maybe_file_size_bytes);
let maybe_mimetype = upload_media_request.file_bytes
.as_ref()
.map(|bytes| get_mimetype_for_bytes(bytes.as_ref()))
.flatten();
let bytes = match upload_media_request.file_bytes {
None => return Err(MediaFileUploadError::BadInput("missing file contents".to_string())),
Some(bytes) => bytes,
};
let hash = sha256_hash_bytes(&bytes)
.map_err(|io_error| {
error!("Problem hashing bytes: {:?}", io_error);
MediaFileUploadError::ServerError
})?;
let file_size_bytes = bytes.len();
let mut maybe_duration_millis = None;
let mut maybe_codec_name = None;
let mut media_file_type = None;
if let Some(mimetype) = maybe_mimetype.as_deref() {
if !allowed_mimetypes.contains(mimetype) {
// NB: Don't let our error message inject malicious strings
let filtered_mimetype = mimetype
.chars()
.filter(|c| c.is_ascii())
.filter(|c| c.is_alphanumeric() || *c == '/')
.collect::<String>();
return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype)));
}
// NB: .aiff (audio/aiff) isn't supported by Symphonia:
// It contains uncompressed PCM-encoded audio similar to wav.
// See: https://github.com/pdeljanov/Symphonia/issues/75
// NB: The following formats are not supported by Symphonia and
// do not have any open issues filed. They may simply be too old:
// - .wma (audio/x-ms-wma)
// - .avi (video/x-msvideo)
media_file_type = match mimetype {
// Audio
"audio/aac" /* .aac */ => Some(MediaFileType::Audio),
"audio/m4a" /* .m4a */ => Some(MediaFileType::Audio),
"audio/mpeg" /* .mp3 */ => Some(MediaFileType::Audio),
"audio/ogg" /* .ogg */ => Some(MediaFileType::Audio),
"audio/opus" /* .opus */ => Some(MediaFileType::Audio),
"audio/x-flac" /* .flac */ => Some(MediaFileType::Audio),
"audio/x-wav" /* .wav */ => Some(MediaFileType::Audio),
// Image
"image/jpeg" /* .jpg */ => Some(MediaFileType::Image),
"image/png" /* .png */ => Some(MediaFileType::Image),
"image/webp" /* .webp */ => Some(MediaFileType::Image),
// Video
"video/mp4" /* .mp4 */ => Some(MediaFileType::Video),
"video/quicktime" /* .mov */ => Some(MediaFileType::Video),
"video/webm" /* .webm */ => Some(MediaFileType::Video),
_ => None,
};
let do_audio_decode = match mimetype {
// TODO: Revisit when Safari can send us this metadata consistently
"audio/mp4" | "video/mp4" => false,
"audio/opus" => {
// TODO/FIXME(bt, 2023-05-19): Symphonia is currently broken for Firefox's opus.
// We're on an off-master branch that may resolve the problem in the future, but for now
// it panics as follows:
//
// [2023-05-19T10:25:34Z INFO symphonia_core::probe] found a possible format marker within [4f, 67, 67, 53, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, d4, c5] @ 0+2 bytes.
// [2023-05-19T10:25:34Z INFO symphonia_core::probe] found the format marker [4f, 67, 67, 53] @ 0+2 bytes.
// [2023-05-19T10:25:34Z DEBUG symphonia_format_ogg::page] grow page buffer to 8192 bytes
// [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] starting new physical stream
// [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] selected opus mapper for stream with serial=0x19aac5d4
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Probed!
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Find audio track...
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Found audio track (maybe)
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe track duration: None
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe codec short name: Some("opus")
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Opus handler
// [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Media source stream
// [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : options...
// [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : try_read...
// [2023-05-19T10:25:34Z DEBUG symphonia_format_mkv::ebml] element with tag: 4F67
// thread 'actix-rt|system:0|arbiter:1' panicked at 'assertion failed: `(left == right)`
// left: `Unknown`,
// right: `Ebml`: EBML element type must be checked before calling this function', /Users/bt/.cargo/git/checkouts/symphonia-8fbe6c90fc095688/e1a7009/symphonia-format-mkv/src/ebml.rs:335:9
false
}
// Also, don't decode images
"image/jpeg" => false,
"image/png" => false,
"image/webp" => false,
_ => true,
};
if do_audio_decode && media_file_type.is_some() {
let basic_info = decode_basic_audio_bytes_info(
bytes.as_ref(),
Some(mimetype),
None
).map_err(|e| {
warn!("file decoding error: {:?}", e);
MediaFileUploadError::BadInput("could not decode file".to_string())
})?;
maybe_duration_millis = basic_info.duration_millis;
maybe_codec_name = basic_info.codec_name;
}
}
let media_file_type = match media_file_type {
Some(m) => m,
None => {
warn!("Invalid mimetype: {:?}", maybe_mimetype);
return Err(MediaFileUploadError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype)))
},
};
let upload_type = match upload_media_request.media_source {
MediaFileUploadSource::Unknown => UploadType::Filesystem,
MediaFileUploadSource::UserFile => UploadType::Filesystem,
MediaFileUploadSource::UserDeviceApi => UploadType::DeviceCaptureApi,
};
// TODO: Clean up code
let mime_type = match maybe_mimetype {
Some(m) => m,
None => {
warn!("Missing mimetype!");
return Err(MediaFileUploadError::BadInput("Missing mimetype".to_string()));
},
};
// TODO(bt,2023-12-20): Reasonable extensions.
let extension = match mime_type {
// Audio
"audio/aac" /* .aac */ => Some(".aac"),
"audio/m4a" /* .m4a */ => Some(".m4a"),
"audio/mpeg" /* .mp3 */ => Some(".mp3"),
"audio/ogg" /* .ogg */ => Some(".ogg"),
"audio/opus" /* .opus */ => Some(".opus"),
"audio/x-flac" /* .flac */ => Some(".flac"),
"audio/x-wav" /* .wav */ => Some(".wav"),
// Image
"image/jpeg" /* .jpg */ => Some(".jpg"),
"image/png" /* .png */ => Some(".png"),
"image/webp" /* .webp */ => Some(".webp"),
// Video
"video/mp4" /* .mp4 */ => Some(".mp4"),
"video/quicktime" /* .mov */ => Some(".mov"),
"video/webm" /* .webm */ => Some(".webm"),
_ => None,
};
const PREFIX : Option<&str> = Some("upload_");
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, extension);
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
server_state.public_bucket_client.upload_file_with_content_type(
public_upload_path.get_full_object_path_str(),
bytes.as_ref(),
mime_type)
.await
.map_err(|e| {
warn!("Upload media bytes to bucket error: {:?}", e);
MediaFileUploadError::ServerError
})?;
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility,
upload_type,
media_file_type,
maybe_mime_type: Some(mime_type),
file_size_bytes: file_size_bytes as u64,
duration_millis: 0, // TODO
sha256_checksum: &hash,
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: extension,
pool: &server_state.mysql_pool,
})
.await
.map_err(|err| {
warn!("New generic download creation DB error: {:?}", err);
MediaFileUploadError::ServerError
})?;
info!("new media file id: {} token: {:?}", record_id, &token);
Ok(SuccessCase::MediaSuccessfullyUploaded {
media_file_token: token,
})
}
@@ -0,0 +1,36 @@
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::HttpResponse;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
#[derive(Debug, Serialize)]
pub enum MediaFileUploadError {
BadInput(String),
NotAuthorized,
MustBeLoggedIn,
ServerError,
RateLimited,
}
impl ResponseError for MediaFileUploadError {
fn status_code(&self) -> StatusCode {
match *self {
MediaFileUploadError::BadInput(_) => StatusCode::BAD_REQUEST,
MediaFileUploadError::NotAuthorized => StatusCode::UNAUTHORIZED,
MediaFileUploadError::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
MediaFileUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
MediaFileUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
impl std::fmt::Display for MediaFileUploadError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
@@ -0,0 +1,64 @@
use std::collections::HashSet;
use std::sync::Arc;
use actix_multipart::Multipart;
use actix_web::{HttpRequest, HttpResponse, web};
use once_cell::sync::Lazy;
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upload::process_upload_media_file::process_upload_media_file;
use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
use crate::server_state::ServerState;
#[derive(Serialize)]
pub struct UploadMediaSuccessResponse {
pub success: bool,
pub media_file_token: MediaFileToken,
}
static ALLOWED_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
HashSet::from([
// Audio
"audio/aac",
"audio/m4a",
"audio/mpeg",
"audio/ogg",
"audio/opus",
"audio/x-flac",
"audio/x-wav",
// Mixed
"audio/mp4", // iPhone seems to upload these as audio
// Video
"video/mp4",
"video/webm",
//"video/quicktime",
])
});
pub async fn upload_media_file_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
mut multipart_payload: Multipart,
) -> Result<HttpResponse, MediaFileUploadError> {
let response = process_upload_media_file(
&http_request,
&server_state,
multipart_payload,
&ALLOWED_MIME_TYPES).await?;
let media_file_token = response.to_media_token();
let response = UploadMediaSuccessResponse {
success: true,
media_file_token,
};
let body = serde_json::to_string(&response)
.map_err(|e| MediaFileUploadError::ServerError)?;
return Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body));
}
@@ -10,6 +10,7 @@ use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use log::warn;
use utoipa::ToSchema;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs};
@@ -28,19 +29,19 @@ use tokens::tokens::zs_voices::ZsVoiceToken;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct CreateUserBookmarkRequest {
entity_token: String,
entity_type: UserBookmarkEntityType,
}
#[derive(Serialize)]
#[derive(Serialize, ToSchema)]
pub struct CreateUserBookmarkSuccessResponse {
pub success: bool,
pub user_bookmark_token: UserBookmarkToken,
}
#[derive(Debug)]
#[derive(Debug, ToSchema)]
pub enum CreateUserBookmarkError {
BadInput(String),
NotAuthorized,
@@ -74,6 +75,15 @@ impl fmt::Display for CreateUserBookmarkError {
}
}
#[utoipa::path(
post,
path = "/v1/user_bookmarks/create",
request_body = CreateUserBookmarkRequest,
responses(
(status = 200, body = CreateUserBookmarkSuccessResponse),
(status = 400, body = CreateUserBookmarkError),
)
)]
pub async fn create_user_bookmark_handler(
http_request: HttpRequest,
request: web::Json<CreateUserBookmarkRequest>,
@@ -11,6 +11,7 @@ use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Path;
use log::warn;
use utoipa::ToSchema;
use mysql_queries::queries::user_bookmarks::delete_user_bookmark::delete_user_bookmark;
use tokens::tokens::user_bookmarks::UserBookmarkToken;
@@ -20,18 +21,18 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success
use crate::server_state::ServerState;
/// For the URL PathInfo
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct DeleteUserBookmarkPathInfo {
user_bookmark_token: UserBookmarkToken,
}
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct DeleteUserBookmarkRequest {
/// NB: this is only to disambiguate when a user is both a mod and an author.
as_mod: Option<bool>,
}
#[derive(Debug)]
#[derive(Debug, ToSchema)]
pub enum DeleteUserBookmarkError {
BadInput(String),
NotAuthorized,
@@ -68,6 +69,20 @@ impl fmt::Display for DeleteUserBookmarkError {
}
}
#[utoipa::path(
delete,
path = "/v1/user_bookmarks/delete/{user_bookmark_token}",
params(
("user_bookmark_token", description = "UserBookmarkToken"),
),
responses(
(status = 200, description = "Delete User Bookmark", body = SimpleGenericJsonSuccess),
(status = 400, description = "Bad input", body = DeleteUserBookmarkError),
(status = 401, description = "Not authorized", body = DeleteUserBookmarkError),
(status = 404, description = "Not found", body = DeleteUserBookmarkError),
(status = 500, description = "Server error", body = DeleteUserBookmarkError),
),
)]
pub async fn delete_user_bookmark_handler(
http_request: HttpRequest,
path: Path<DeleteUserBookmarkPathInfo>,
@@ -12,6 +12,7 @@ use actix_web::http::StatusCode;
use actix_web::web::Path;
use chrono::{DateTime, Utc};
use log::warn;
use utoipa::ToSchema;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken;
@@ -23,20 +24,20 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
/// For the URL PathInfo
#[derive(Deserialize)]
pub struct ListUserBookmarkPathInfo {
#[derive(Deserialize, ToSchema)]
pub struct ListUserBookmarksForEntityPathInfo {
entity_type: UserBookmarkEntityType,
entity_token: String,
}
#[derive(Serialize)]
pub struct ListUserBookmarksSuccessResponse {
#[derive(Serialize, ToSchema)]
pub struct ListUserBookmarksForEntitySuccessResponse {
pub success: bool,
pub user_bookmarks: Vec<UserBookmark>,
pub user_bookmarks: Vec<UserBookmarkForEntityListItem>,
}
#[derive(Serialize)]
pub struct UserBookmark {
#[derive(Serialize, ToSchema)]
pub struct UserBookmarkForEntityListItem {
pub token: UserBookmarkToken,
pub user: UserDetailsLight,
@@ -45,21 +46,21 @@ pub struct UserBookmark {
pub updated_at: DateTime<Utc>,
}
#[derive(Debug)]
pub enum ListUserBookmarksError {
#[derive(Debug, ToSchema)]
pub enum ListUserBookmarksForEntityError {
ServerError,
}
impl ResponseError for ListUserBookmarksError {
impl ResponseError for ListUserBookmarksForEntityError {
fn status_code(&self) -> StatusCode {
match *self {
ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
ListUserBookmarksForEntityError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
ListUserBookmarksError::ServerError => "server error".to_string(),
ListUserBookmarksForEntityError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
@@ -67,17 +68,30 @@ impl ResponseError for ListUserBookmarksError {
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for ListUserBookmarksError {
impl fmt::Display for ListUserBookmarksForEntityError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[utoipa::path(
get,
path = "/v1/user_bookmarks/list/{entity_type}/{entity_token}",
params(
("entity_type", description="The type of entity to list bookmarks for"),
("entity_token", description="The token of the entity to list bookmarks for"),
),
responses(
(status = 200, body = ListUserBookmarksForEntitySuccessResponse),
(status = 500, body = ListUserBookmarksForEntityError),
),
)]
pub async fn list_user_bookmarks_for_entity_handler(
_http_request: HttpRequest,
path: Path<ListUserBookmarkPathInfo>,
path: Path<ListUserBookmarksForEntityPathInfo>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ListUserBookmarksError>
) -> Result<HttpResponse, ListUserBookmarksForEntityError>
{
let entity_token = UserBookmarkEntityToken::from_entity_type_and_token(
path.entity_type, &path.entity_token);
@@ -91,14 +105,14 @@ pub async fn list_user_bookmarks_for_entity_handler(
Ok(results) => results,
Err(e) => {
warn!("Query error: {:?}", e);
return Err(ListUserBookmarksError::ServerError);
return Err(ListUserBookmarksForEntityError::ServerError);
}
};
let response = ListUserBookmarksSuccessResponse {
let response = ListUserBookmarksForEntitySuccessResponse {
success: true,
user_bookmarks: user_bookmarks.into_iter()
.map(|user_bookmark| UserBookmark {
.map(|user_bookmark| UserBookmarkForEntityListItem {
token: user_bookmark.token,
user: UserDetailsLight {
user_token: user_bookmark.user_token.clone(),
@@ -114,7 +128,7 @@ pub async fn list_user_bookmarks_for_entity_handler(
};
let body = serde_json::to_string(&response)
.map_err(|_e| ListUserBookmarksError::ServerError)?;
.map_err(|_e| ListUserBookmarksForEntityError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -12,6 +12,7 @@ use actix_web::http::StatusCode;
use actix_web::web::{Path, Query};
use chrono::{DateTime, Utc};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type};
@@ -20,34 +21,34 @@ use tokens::tokens::user_bookmarks::UserBookmarkToken;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct ListUserBookmarksPathInfo {
username: String,
}
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema, IntoParams)]
pub struct ListUserBookmarksQueryData {
maybe_scoped_entity_type: Option<UserBookmarkEntityType>,
}
#[derive(Serialize)]
pub struct ListUserBookmarksSuccessResponse {
#[derive(Serialize, ToSchema)]
pub struct ListUserBookmarksForUserSuccessResponse {
pub success: bool,
pub user_bookmarks: Vec<UserBookmark>,
pub user_bookmarks: Vec<UserBookmarkListItem>,
}
#[derive(Serialize)]
pub struct UserBookmark {
#[derive(Serialize, ToSchema)]
pub struct UserBookmarkListItem {
pub token: UserBookmarkToken,
pub details: UserBookmarkDetails,
pub details: UserBookmarkDetailsForUserList,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Serialize)]
pub struct UserBookmarkDetails {
#[derive(Serialize, ToSchema)]
pub struct UserBookmarkDetailsForUserList {
// TODO: This needs titles or some other summary metadata.
pub entity_type: UserBookmarkEntityType,
pub entity_token: String,
@@ -59,21 +60,21 @@ pub struct UserBookmarkDetails {
pub maybe_thumbnail_url: Option<String>,
}
#[derive(Debug)]
pub enum ListUserBookmarksError {
#[derive(Debug, ToSchema)]
pub enum ListUserBookmarksForUserError {
ServerError,
}
impl ResponseError for ListUserBookmarksError {
impl ResponseError for ListUserBookmarksForUserError {
fn status_code(&self) -> StatusCode {
match *self {
ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
ListUserBookmarksForUserError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
ListUserBookmarksError::ServerError => "server error".to_string(),
ListUserBookmarksForUserError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
@@ -81,18 +82,30 @@ impl ResponseError for ListUserBookmarksError {
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for ListUserBookmarksError {
impl fmt::Display for ListUserBookmarksForUserError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[utoipa::path(
get,
path = "/v1/user_bookmarks/list/user/{username}",
params(
("username", description = "The username of the user whose bookmarks to list."),
ListUserBookmarksQueryData
),
responses(
(status = 200, description = "List User Bookmarks", body = ListUserBookmarksForUserSuccessResponse),
(status = 500, description = "Server error", body = ListUserBookmarksForUserError),
),
)]
pub async fn list_user_bookmarks_for_user_handler(
_http_request: HttpRequest,
path: Path<ListUserBookmarksPathInfo>,
query: Query<ListUserBookmarksQueryData>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ListUserBookmarksError>
) -> Result<HttpResponse, ListUserBookmarksForUserError>
{
let query_results = match query.maybe_scoped_entity_type {
@@ -106,16 +119,16 @@ pub async fn list_user_bookmarks_for_user_handler(
Ok(results) => results,
Err(e) => {
warn!("Query error: {:?}", e);
return Err(ListUserBookmarksError::ServerError);
return Err(ListUserBookmarksForUserError::ServerError);
}
};
let response = ListUserBookmarksSuccessResponse {
let response = ListUserBookmarksForUserSuccessResponse {
success: true,
user_bookmarks: user_bookmarks.into_iter()
.map(|user_bookmark| UserBookmark {
.map(|user_bookmark| UserBookmarkListItem {
token: user_bookmark.token,
details: UserBookmarkDetails {
details: UserBookmarkDetailsForUserList {
entity_type: user_bookmark.entity_type,
entity_token: user_bookmark.entity_token,
maybe_summary_text: user_bookmark.maybe_entity_descriptive_text,
@@ -130,7 +143,7 @@ pub async fn list_user_bookmarks_for_user_handler(
};
let body = serde_json::to_string(&response)
.map_err(|_e| ListUserBookmarksError::ServerError)?;
.map_err(|_e| ListUserBookmarksForUserError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -14,7 +14,6 @@ use futures::TryStreamExt;
use log::{error, info, warn};
use r2d2_redis::redis::Commands;
use buckets::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
use container_common::token::random_uuid::generate_random_uuid;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_ip::get_request_ip;
@@ -252,13 +251,15 @@ pub async fn enqueue_infer_w2l_with_uploads(
let audio_upload_bucket_hash = upload_uuid.clone();
let audio_upload_bucket_path = hash_to_bucket_path_string(
&upload_uuid,
Some(&server_state.audio_uploads_bucket_root)
).map_err(|e| {
warn!("Hash bucket path error: {:?}", e);
InferW2lWithUploadError::ServerError
})?;
//let audio_upload_bucket_path = hash_to_bucket_path_string(
// &upload_uuid,
// Some(&server_state.audio_uploads_bucket_root)
//).map_err(|e| {
// warn!("Hash bucket path error: {:?}", e);
// InferW2lWithUploadError::ServerError
//})?;
let audio_upload_bucket_path = "THIS_IS_BROKEN_BECAUSE_W2L_IS_DEAD";
info!("Uploading audio to bucket...");
server_state.private_bucket_client.upload_file_with_content_type(

Some files were not shown because too many files have changed in this diff Show More