Compare commits

..
Author SHA1 Message Date
Michael Chung 30127e19e9 Fixed added weights type by request. 2023-12-23 23:25:43 -05:00
Michael Chung c50a2b36aa Add missing data in response
get weight endpoint to have updated_at (the list weight has but the single weight doesnt)
profile weight list endpoint weights_category is missing from the response, i need it for rendering the cards by what type of weight it is.
2023-12-23 14:46:45 -05:00
ArEnSc 49796c69e3 Merge pull request #239 from storytold/feature/weights_files
Feature/weights files
2023-12-21 20:51:39 -05:00
ArEnSc 08f59a1989 Update .env 2023-12-21 19:53:07 -05:00
Michael Chung 15c218286f Bucket Orchestration work complete. 2023-12-21 19:51:32 -05:00
Brandon Thomas 268346f662 delete more easily 2023-12-21 19:40:33 -05:00
Brandon Thomas 7897ef730e refactor, fix tests 2023-12-21 19:04:56 -05:00
Brandon Thomas 43da5c91ad fix compile error 2023-12-21 18:47:28 -05:00
Brandon Thomas 71360879f3 cleanup 2023-12-21 18:43:18 -05:00
ArEnSc 7f80e106bd Merge pull request #238 from storytold/dec-21-mithoo
use public pagination for profile weights list endpoint
2023-12-21 14:30:17 -05:00
Kasisnu 58181b32cf use public pagination for profile weights list endpoint 2023-12-22 00:45:24 +05:30
Michael Chung 0efe2d8177 Working Tests 2023-12-21 12:38:04 -05:00
ArEnSc 62c5d86584 Merge pull request #237 from storytold/dec-21-mithoo
Document user bookmarks endpoints
2023-12-21 12:34:48 -05:00
Brooks Palin cbb950cb40 Update general_setup.md
Fixed path for for storyteller-web
2023-12-21 10:30:36 -07:00
Kasisnu 64be9e49af Document user bookmarks endpoints 2023-12-21 19:52:44 +05:30
Brandon Thomas bd688703df Merge pull request #236 from storytold/brandon-creator-details
add creator details
2023-12-21 04:29:36 -05:00
Brandon Thomas c627c13e2d add creator details 2023-12-21 04:28:55 -05:00
Brandon Thomas 84104d91ac Merge pull request #235 from storytold/brandon-model-weight-avatars
model weight avatars
2023-12-21 04:13:58 -05:00
Brandon Thomas 64806135c5 model weight avatars 2023-12-21 04:13:44 -05:00
Brandon Thomas 5e48a6193a reorganize modules 2023-12-21 02:57:00 -05:00
Brandon Thomas 11c17f0a1d fix lints 2023-12-21 02:37:29 -05:00
Michael Chung ed97cdbb0d Ok builds again going to run unit tests. 2023-12-21 02:25:20 -05:00
Brandon Thomas cd91a25c3d Merge pull request #219 from storytold/dec-18-mithoo
document media files list endpoints, fix public page offset bug
2023-12-21 02:11:19 -05:00
Michael Chung 6b2d4ffdce Adding the mocks 2023-12-21 01:58:43 -05:00
Michael Chung 86c80fdf19 Things build again editor works well. 2023-12-21 01:01:35 -05:00
Michael Chung eca8b01f14 Missed 1 file. 2023-12-21 00:42:22 -05:00
Michael Chung 1bddb21e9f Missed 1 file. 2023-12-21 00:41:26 -05:00
Michael Chung 20b950794c Ok it builds need to test it using a mock. 2023-12-21 00:40:17 -05:00
Michael Chung 0d58176f0f Minor bug fixes then testing. 2023-12-20 02:23:45 -05:00
Michael Chung 57f383612f Now testing time! 2023-12-20 02:20:44 -05:00
Michael Chung ab3a312898 Ok so it actually compiles right now. 2023-12-20 00:13:02 -05:00
Michael Chung a2aa71e8a6 builds properly now working on the download aspect. 2023-12-19 23:25:18 -05:00
Michael Chung 3e02d791af Merge branch 'master' into feature/weights_files 2023-12-19 17:44:39 -05:00
Michael Chung 70694ce330 Need to replace the hashing function with a new one and merge master. 2023-12-19 17:43:24 -05:00
Michael Chung 775d867add Ok fixed compilation issues. 2023-12-19 03:05:39 -05:00
Michael Chung da62443522 Fixed some issues using a box 2023-12-19 02:48:41 -05:00
Michael Chung 1ae2df8fa7 working on the file manager for uploads and downloads 2023-12-19 02:40:42 -05:00
Michael Chung 2e6d7d96e4 Merge branch 'master' into feature/weights_files 2023-12-18 21:18:01 -05:00
Michael Chung 39bae02259 Merge branch 'feature/weights_files' of https://github.com/storytold/storyteller-rust into feature/weights_files 2023-12-18 09:26:42 -05:00
Michael Chung 7755a5683b Working on file descriptors though gotta learn some rust things, quickly for boiler plate. 2023-12-18 09:26:12 -05:00
82 changed files with 1698 additions and 389 deletions
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.public_bucket_hash,\n wt.maybe_public_bucket_prefix,\n wt.maybe_public_bucket_extension,\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n WHERE\n wt.token = ?\n AND wt.user_deleted_at IS NULL\n AND wt.mod_deleted_at IS NULL\n ",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.public_bucket_hash,\n wt.maybe_public_bucket_prefix,\n wt.maybe_public_bucket_extension,\n\n avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,\n avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,\n avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,\n\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n JOIN users\n ON users.token = wt.creator_user_token\n LEFT OUTER JOIN media_files as avatar\n ON avatar.token = wt.maybe_avatar_media_file_token\n WHERE\n wt.token = ?\n ",
"describe": {
"columns": [
{
@@ -85,6 +85,36 @@
},
{
"ordinal": 8,
"name": "creator_username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 9,
"name": "creator_display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 10,
"name": "creator_gravatar_hash",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 11,
"name": "creator_ip_address",
"type_info": {
"type": "VarString",
@@ -94,7 +124,7 @@
}
},
{
"ordinal": 9,
"ordinal": 12,
"name": "creator_set_visibility: enums::common::visibility::Visibility",
"type_info": {
"type": "String",
@@ -104,7 +134,7 @@
}
},
{
"ordinal": 10,
"ordinal": 13,
"name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
@@ -114,7 +144,7 @@
}
},
{
"ordinal": 11,
"ordinal": 14,
"name": "original_download_url",
"type_info": {
"type": "VarString",
@@ -124,7 +154,7 @@
}
},
{
"ordinal": 12,
"ordinal": 15,
"name": "original_filename",
"type_info": {
"type": "VarString",
@@ -134,7 +164,7 @@
}
},
{
"ordinal": 13,
"ordinal": 16,
"name": "file_size_bytes",
"type_info": {
"type": "Long",
@@ -144,7 +174,7 @@
}
},
{
"ordinal": 14,
"ordinal": 17,
"name": "file_checksum_sha2",
"type_info": {
"type": "String",
@@ -154,7 +184,7 @@
}
},
{
"ordinal": 15,
"ordinal": 18,
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
@@ -164,7 +194,7 @@
}
},
{
"ordinal": 16,
"ordinal": 19,
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
@@ -174,7 +204,7 @@
}
},
{
"ordinal": 17,
"ordinal": 20,
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
@@ -184,7 +214,37 @@
}
},
{
"ordinal": 18,
"ordinal": 21,
"name": "maybe_avatar_public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 22,
"name": "maybe_avatar_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 23,
"name": "maybe_avatar_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 24,
"name": "cached_user_ratings_negative_count",
"type_info": {
"type": "Long",
@@ -194,7 +254,7 @@
}
},
{
"ordinal": 19,
"ordinal": 25,
"name": "cached_user_ratings_positive_count",
"type_info": {
"type": "Long",
@@ -204,7 +264,7 @@
}
},
{
"ordinal": 20,
"ordinal": 26,
"name": "cached_user_ratings_total_count",
"type_info": {
"type": "Long",
@@ -214,7 +274,7 @@
}
},
{
"ordinal": 21,
"ordinal": 27,
"name": "maybe_cached_user_ratings_ratio",
"type_info": {
"type": "Float",
@@ -224,7 +284,7 @@
}
},
{
"ordinal": 22,
"ordinal": 28,
"name": "cached_user_ratings_last_updated_at",
"type_info": {
"type": "Timestamp",
@@ -234,7 +294,7 @@
}
},
{
"ordinal": 23,
"ordinal": 29,
"name": "version",
"type_info": {
"type": "Long",
@@ -244,7 +304,7 @@
}
},
{
"ordinal": 24,
"ordinal": 30,
"name": "created_at",
"type_info": {
"type": "Timestamp",
@@ -254,7 +314,7 @@
}
},
{
"ordinal": 25,
"ordinal": 31,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
@@ -264,7 +324,7 @@
}
},
{
"ordinal": 26,
"ordinal": 32,
"name": "user_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -274,7 +334,7 @@
}
},
{
"ordinal": 27,
"ordinal": 33,
"name": "mod_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -298,6 +358,9 @@
false,
false,
false,
false,
false,
false,
true,
true,
true,
@@ -306,6 +369,9 @@
false,
true,
true,
true,
true,
true,
false,
false,
false,
@@ -318,5 +384,5 @@
true
]
},
"hash": "7434b13f252c43b854e51c7761a71842076694c31cfc9336863638f829388727"
"hash": "4e30e34cdf7a5834a3981d5e634506966bac9f9c420e800d15ce313778272cee"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.public_bucket_hash,\n wt.maybe_public_bucket_prefix,\n wt.maybe_public_bucket_extension,\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n WHERE\n wt.token = ?\n ",
"query": "\n SELECT\n wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n wt.title,\n wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n wt.maybe_thumbnail_token,\n wt.description_markdown,\n wt.description_rendered_html,\n wt.creator_ip_address,\n wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n\n wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n users.username as creator_username,\n users.display_name as creator_display_name,\n users.email_gravatar_hash AS creator_gravatar_hash,\n\n wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n wt.original_download_url,\n wt.original_filename,\n wt.file_size_bytes,\n wt.file_checksum_sha2,\n wt.public_bucket_hash,\n wt.maybe_public_bucket_prefix,\n wt.maybe_public_bucket_extension,\n\n avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,\n avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,\n avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,\n\n wt.cached_user_ratings_negative_count,\n wt.cached_user_ratings_positive_count,\n wt.cached_user_ratings_total_count,\n wt.maybe_cached_user_ratings_ratio,\n wt.cached_user_ratings_last_updated_at,\n wt.version,\n wt.created_at,\n wt.updated_at,\n wt.user_deleted_at,\n wt.mod_deleted_at\n FROM model_weights as wt\n JOIN users\n ON users.token = wt.creator_user_token\n LEFT OUTER JOIN media_files as avatar\n ON avatar.token = wt.maybe_avatar_media_file_token\n WHERE\n wt.token = ?\n AND wt.user_deleted_at IS NULL\n AND wt.mod_deleted_at IS NULL\n ",
"describe": {
"columns": [
{
@@ -75,16 +75,6 @@
},
{
"ordinal": 7,
"name": "creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 8,
"name": "creator_ip_address",
"type_info": {
"type": "VarString",
@@ -94,7 +84,7 @@
}
},
{
"ordinal": 9,
"ordinal": 8,
"name": "creator_set_visibility: enums::common::visibility::Visibility",
"type_info": {
"type": "String",
@@ -103,8 +93,48 @@
"max_size": 28
}
},
{
"ordinal": 9,
"name": "creator_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 10,
"name": "creator_username",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 11,
"name": "creator_display_name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 80
}
},
{
"ordinal": 12,
"name": "creator_gravatar_hash",
"type_info": {
"type": "String",
"flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 13,
"name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
"type_info": {
"type": "VarString",
@@ -114,7 +144,7 @@
}
},
{
"ordinal": 11,
"ordinal": 14,
"name": "original_download_url",
"type_info": {
"type": "VarString",
@@ -124,7 +154,7 @@
}
},
{
"ordinal": 12,
"ordinal": 15,
"name": "original_filename",
"type_info": {
"type": "VarString",
@@ -134,7 +164,7 @@
}
},
{
"ordinal": 13,
"ordinal": 16,
"name": "file_size_bytes",
"type_info": {
"type": "Long",
@@ -144,7 +174,7 @@
}
},
{
"ordinal": 14,
"ordinal": 17,
"name": "file_checksum_sha2",
"type_info": {
"type": "String",
@@ -154,7 +184,7 @@
}
},
{
"ordinal": 15,
"ordinal": 18,
"name": "public_bucket_hash",
"type_info": {
"type": "VarString",
@@ -164,7 +194,7 @@
}
},
{
"ordinal": 16,
"ordinal": 19,
"name": "maybe_public_bucket_prefix",
"type_info": {
"type": "VarString",
@@ -174,7 +204,7 @@
}
},
{
"ordinal": 17,
"ordinal": 20,
"name": "maybe_public_bucket_extension",
"type_info": {
"type": "VarString",
@@ -184,7 +214,37 @@
}
},
{
"ordinal": 18,
"ordinal": 21,
"name": "maybe_avatar_public_bucket_hash",
"type_info": {
"type": "VarString",
"flags": "BINARY | NO_DEFAULT_VALUE",
"char_set": 224,
"max_size": 128
}
},
{
"ordinal": 22,
"name": "maybe_avatar_public_bucket_prefix",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 23,
"name": "maybe_avatar_public_bucket_extension",
"type_info": {
"type": "VarString",
"flags": "BINARY",
"char_set": 224,
"max_size": 64
}
},
{
"ordinal": 24,
"name": "cached_user_ratings_negative_count",
"type_info": {
"type": "Long",
@@ -194,7 +254,7 @@
}
},
{
"ordinal": 19,
"ordinal": 25,
"name": "cached_user_ratings_positive_count",
"type_info": {
"type": "Long",
@@ -204,7 +264,7 @@
}
},
{
"ordinal": 20,
"ordinal": 26,
"name": "cached_user_ratings_total_count",
"type_info": {
"type": "Long",
@@ -214,7 +274,7 @@
}
},
{
"ordinal": 21,
"ordinal": 27,
"name": "maybe_cached_user_ratings_ratio",
"type_info": {
"type": "Float",
@@ -224,7 +284,7 @@
}
},
{
"ordinal": 22,
"ordinal": 28,
"name": "cached_user_ratings_last_updated_at",
"type_info": {
"type": "Timestamp",
@@ -234,7 +294,7 @@
}
},
{
"ordinal": 23,
"ordinal": 29,
"name": "version",
"type_info": {
"type": "Long",
@@ -244,7 +304,7 @@
}
},
{
"ordinal": 24,
"ordinal": 30,
"name": "created_at",
"type_info": {
"type": "Timestamp",
@@ -254,7 +314,7 @@
}
},
{
"ordinal": 25,
"ordinal": 31,
"name": "updated_at",
"type_info": {
"type": "Timestamp",
@@ -264,7 +324,7 @@
}
},
{
"ordinal": 26,
"ordinal": 32,
"name": "user_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -274,7 +334,7 @@
}
},
{
"ordinal": 27,
"ordinal": 33,
"name": "mod_deleted_at",
"type_info": {
"type": "Timestamp",
@@ -298,6 +358,9 @@
false,
false,
false,
false,
false,
false,
true,
true,
true,
@@ -306,6 +369,9 @@
false,
true,
true,
true,
true,
true,
false,
false,
false,
@@ -318,5 +384,5 @@
true
]
},
"hash": "964cda4c4505907fd8af46b2f6578a62ab5bba7cc5534469c3c061f9cfb0cd50"
"hash": "93084dc42ae4ada186dc90bdd470122342586f4f19a3ac961286c1c54a5c32b2"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\n UPDATE model_weights\n SET\n maybe_avatar_media_file_token = ?\n WHERE token = ?\n LIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 2
},
"nullable": []
},
"hash": "bda002b24b98328f0fb0440cfc82a47faf01e5d16d1030e15a7e2da161dd0d80"
}
Generated
+11 -2
View File
@@ -620,9 +620,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.74"
version = "0.1.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98"
dependencies = [
"proc-macro2",
"quote",
@@ -1124,10 +1124,19 @@ name = "cloud_storage"
version = "0.0.1"
dependencies = [
"anyhow",
"async-trait",
"buckets",
"crockford",
"easyenv",
"env_logger",
"errors",
"filesys",
"hashing",
"log",
"mimetypes",
"my-workspace-hack",
"rust-s3",
"storyteller_root",
"tokio",
]
@@ -310,7 +310,7 @@ Secrets:
- Put `.env-secrets` in the root `storyteller-rust` directory.
- Per app secrets (ask Brandon for these):
- Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller-web/config`
- Put `storteller-web.development-secrets.env` in `crates/service/web/storyteller_web/config`
- Put `inference-job.development-secrets.env` in `crates/service/job/inference_job/config`
- Put `donwload-job.development-secrets.env` in `crates/service/job/download_job/config`
@@ -392,4 +392,4 @@ We use this: `https://github.com/juhaku/utoipa`
The documentation example is in:
`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
Write into the api doc to:
`crates/service/web/storyteller_web/src/api_doc.rs`
`crates/service/web/storyteller_web/src/api_doc.rs`
+1
View File
@@ -18,6 +18,7 @@ path = "src/lib.rs"
crockford = { path = "../../lib/crockford" }
errors = { path = "../../lib/errors" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+1 -1
View File
@@ -10,7 +10,7 @@
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
// #![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
@@ -1,3 +0,0 @@
pub trait descriptor {
fn my_method(&self);
}
@@ -1,4 +0,0 @@
pub mod bucket_directory;
pub mod bucket_file_path;
pub mod web_file_manager;
@@ -1,32 +0,0 @@
pub struct FileBucketPath {
pub bucket_clients: BucketClients
}
impl web_file_manager {
pub fn new(my_field: i32) -> Self {
web_file_manager { my_field }
}
pub fn increment(&mut self) {
self.my_field += 1;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_new() {
let my_class = web_file_manager::new(10);
assert_eq!(my_class.my_field, 10);
}
#[test]
fn test_increment() {
let mut my_class = web_file_manager::new(10);
my_class.increment();
assert_eq!(my_class.my_field, 11);
}
}
+1 -1
View File
@@ -3,4 +3,4 @@ pub mod media_uploads;
pub mod public_path;
pub mod voice_conversion_results;
pub mod weight_uploads;
pub mod weight_files;
pub mod weight_files;
@@ -4,10 +4,11 @@ use serde::Serialize;
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
use utoipa::ToSchema;
/// Used in the `user_bookmarks` table in a `VARCHAR(32)` field named `entity_type`.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize, ToSchema)]
pub enum UserBookmarkEntityType {
/// User
#[serde(rename = "user")]
@@ -2,11 +2,12 @@ use std::fmt::Debug;
use serde::Deserialize;
use serde::Serialize;
use utoipa::ToSchema;
use crate::prefixes::TokenPrefix;
/// The primary key for Audit Logs.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
#[sqlx(transparent)]
pub struct UserBookmarkToken(pub String);
@@ -9,7 +9,7 @@ use enums::by_table::model_weights::{
};
use enums::common::visibility::Visibility;
use errors::{anyhow, AnyhowResult};
use mysql_queries::queries::model_weights::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::model_weights::create::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
@@ -5,16 +5,18 @@ use sqlx::{MySql, Pool};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use filesys::file_read_bytes::file_read_bytes;
use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::queries::media_files::create::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use mysql_queries::queries::media_files::get_media_file::get_media_file;
use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use storyteller_root::get_seed_tool_data_root;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::users::UserToken;
@@ -106,7 +108,7 @@ async fn seed_model(
let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
let mut maybe_bucket_prefix = None;
let mut maybe_bucket_extension = None;
if media_file_exists(mysql_pool, media_file_token).await? {
info!("Media file already seeded: {:?}", media_file_token);
return Ok(())
@@ -181,16 +183,16 @@ async fn seed_file_to_bucket(
.extension()
.map(|extension| extension.to_str())
.flatten();
// get the new bucket path ...
let bucket_location = MediaFileBucketPath::generate_new(maybe_bucket_prefix, maybe_bucket_extension);
let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf());
info!("Reading media file: {:?}", media_file_path);
// get meta data
let bytes = file_read_bytes(media_file_path)?;
let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream");
info!("Uploading media file to bucket path: {:?}", bucket_path);
let _r = bucket_clients.public.upload_file_with_content_type(
@@ -15,7 +15,7 @@ use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mysql_queries::queries::model_weights::create_weight::{
use mysql_queries::queries::model_weights::create::create_weight::{
create_weight,
CreateModelWeightsArgs,
};
@@ -25,7 +25,7 @@ use tokens::tokens::users::UserToken;
use crate::bucket_clients::BucketClients;
use crate::seeding::users::HANASHI_USER_TOKEN;
use std::result::Result::Ok;
pub async fn seed_weights_files(
mysql_pool: &Pool<MySql>,
maybe_bucket_clients: Option<&BucketClients>
@@ -202,7 +202,7 @@ async fn seed_file_to_bucket(
.map(|extension| extension.to_str())
.flatten();
// we should just have this be file bucket path and then we can use a file descriptor to turn it into a specific type of path
let bucket_location = WeightFileBucketPath::generate_new(
let bucket_location: WeightFileBucketPath = WeightFileBucketPath::generate_new(
maybe_bucket_prefix,
maybe_bucket_extension
);
+1 -1
View File
@@ -11,7 +11,7 @@ use enums::common::visibility::Visibility;
use errors::{AnyhowResult, bail};
use filesys::file_read_bytes::file_read_bytes;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
use crate::cli_args::parse_cli_args;
@@ -16,14 +16,21 @@ path = "src/lib.rs"
# Internal
buckets = { path = "../../../_types/buckets" }
crockford = { path = "../../../lib/crockford" }
errors = { path = "../../../lib/errors" }
easyenv = { path = "../../../lib/easyenv" }
filesys = { path = "../../../lib/files/filesys" }
hashing = { path = "../../../lib/files/hashing" }
mimetypes = { path = "../../../lib/files/mimetypes" }
storyteller_root = { path = "../../../lib/storyteller_root" }
# Workspace hack (faster compile times)
my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
# External
anyhow = "1.0.75"
log = "0.4.14"
env_logger = "0.10.1"
async-trait = "0.1.75"
# The "never-encode-slash" feature seems to have been removed:
# https://github.com/durch/rust-s3/pull/254
rust-s3 = { version = "0.33.0" }
@@ -31,3 +38,4 @@ rust-s3 = { version = "0.33.0" }
# Runtime
# NB: We switched to Tokio from `async-std` for broader ecosystem support.
tokio = { version = "1.34.0", features = ["macros"] }
@@ -1,6 +1,6 @@
use std::path::Path;
use std::time::Duration;
use errors::AnyhowResult;
use anyhow::anyhow;
use anyhow::bail;
use log::info;
@@ -10,15 +10,31 @@ use s3::creds::Credentials;
use s3::region::Region;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use std::error::Error;
#[derive(Clone)]
pub struct BucketClient {
bucket: Bucket,
/// If set, put all files under this root path.
optional_bucket_root: Option<String>,
}
#[derive(Debug)]
pub enum BucketClientError {
ErrorWithCodeAndMessage { code: u16, message: String },
}
impl std::fmt::Display for BucketClientError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
BucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message),
}
}
}
impl Error for BucketClientError {}
impl BucketClient {
pub fn create(
@@ -87,6 +103,25 @@ impl BucketClient {
Ok(())
}
pub async fn upload_file_with_content_type_process(&self, object_name: &str, bytes: &[u8], content_type: &str) -> AnyhowResult<()> {
info!("Filename for bucket: {}", object_name);
let object_name = self.get_rooted_object_name(object_name);
info!("Rooted filename for bucket: {}", object_name);
let response = self.bucket.put_object_with_content_type(&object_name, bytes, content_type).await?;
let body_bytes = response.bytes();
let code = response.status_code();
info!("upload code: {}", code);
if code != 200 {
let body = String::from_utf8_lossy(body_bytes);
warn!("upload body: {}", body);
return Err(anyhow!("upload failed: {}", code));
} else {
info!("upload success: {}", code);
Ok(())
}
}
#[deprecated = "Use upload_file instead above it returns an error we can surface and act on. upload_file_with_content_type_process"]
pub async fn upload_file_with_content_type(&self, object_name: &str, bytes: &[u8], content_type: &str) -> anyhow::Result<()> {
info!("Filename for bucket: {}", object_name);
@@ -191,8 +226,7 @@ impl BucketClient {
&self,
object_path: P,
filesystem_path: Q,
) -> anyhow::Result<()> {
// TODO I think we might want to return an error here? but there has to a reason why we are just returning ok?
) -> AnyhowResult<()> {
let object_path_str = object_path.as_ref()
.to_str()
.map(|s| s.to_string())
@@ -208,9 +242,7 @@ impl BucketClient {
404 => bail!("File not found in bucket: {}", &object_path_str),
_ => {},
}
info!("download code: {}", status_code);
Ok(())
}
@@ -1,2 +1,4 @@
pub mod bucket_client;
pub mod bucket_path_unifier;
pub mod remote_file_manager;
pub mod remote_file_manager_mock;
@@ -0,0 +1,131 @@
use log::info;
use tokio::time::Duration;
use async_trait::async_trait;
use errors::AnyhowResult;
use crate::bucket_client::BucketClient;
use crate::remote_file_manager::file_meta_data::FileMetaData;
#[async_trait]
pub trait BucketOrchestrationCore {
async fn download_file_to_disk(
&self,
object_path: String,
filesystem_path: String,
is_public: bool,
) -> AnyhowResult<()>;
async fn upload_file_with_content_type_process(&self, object_name: &str,
bytes: &[u8],
content_type: &str,
is_public: bool) -> AnyhowResult<()>;
}
pub struct BucketOrchestration {
access_key: String,
secret_key: String,
region_name: String,
public_bucket_name: String,
private_bucket_name: String,
}
#[async_trait]
impl BucketOrchestrationCore for BucketOrchestration {
async fn download_file_to_disk(
&self,
object_path: String,
filesystem_path: String,
is_public: bool,
) -> AnyhowResult<()> {
let bucket_client = self.get_bucket_with_visbility(is_public).await?;
bucket_client.download_file_to_disk(object_path, filesystem_path).await
}
async fn upload_file_with_content_type_process(&self, object_name: &str,
bytes: &[u8],
content_type: &str,
is_public: bool) -> AnyhowResult<()> {
let bucket_client = self.get_bucket_with_visbility(is_public).await?;
bucket_client.upload_file_with_content_type_process(
object_name,
bytes,
content_type,
).await
}
}
impl BucketOrchestration {
pub fn new_bucket_client_from_existing_env() -> AnyhowResult<Self> {
let access_key = easyenv::get_env_string_required("ACCESS_KEY")?;
let secret_key = easyenv::get_env_string_required("SECRET_KEY")?;
let region_name = easyenv::get_env_string_required("REGION_NAME")?;
let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?;
let bucket_orchestration = BucketOrchestration::new(
access_key,
secret_key,
region_name,
public_bucket_name,
private_bucket_name,
);
Ok(bucket_orchestration)
}
pub fn new(
access_key: String,
secret_key: String,
region_name: String,
public_bucket_name: String,
private_bucket_name: String) -> Self {
Self {
access_key: access_key,
secret_key: secret_key,
region_name: region_name,
public_bucket_name: public_bucket_name,
private_bucket_name: private_bucket_name,
}
}
async fn get_bucket_with_visbility(&self, public: bool) -> AnyhowResult<BucketClient> {
let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
"BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10));
let bucket_client: BucketClient;
if public {
// use public bucket client
info!("Configuring public GCS bucket...");
bucket_client = BucketClient::create(
&self.access_key,
&self.secret_key,
&self.region_name,
&self.public_bucket_name,
None,
Some(bucket_timeout),
)?;
} else {
info!("Configuring private GCS bucket...");
bucket_client = BucketClient::create(
&self.access_key,
&self.secret_key,
&self.region_name,
&self.private_bucket_name,
None,
Some(bucket_timeout),
)?;
}
Ok(bucket_client)
}
}
#[cfg(test)]
mod tests {
#[test]
pub fn test() {}
}
@@ -0,0 +1,35 @@
// Describes the file and how it will be saved to GCBucket
// where the file should be stored based off the descriptor
// DEFAULT IMPLEMENTATION
const REMOTE_FILE_DIRECTORY: &str = "/implement_google_cloud_storage_bucket_name";
// File Descriptor Steers the Bucket Directory
pub trait FileDescriptor {
// By default a file belongs in the public bucket will help us figureout which bucket to use.
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file
// e.g requires! a period .safetensors .bin .jpg
fn get_suffix(&self)->String {
return "implement".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, loRA, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "implement".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
#[cfg(test)]
mod tests {
#[test]
pub fn test() {}
}
@@ -0,0 +1,83 @@
use std::path::PathBuf;
use crockford::crockford_entropy_lower;
use buckets::util::hashed_directory_path_long_string;
use crate::remote_file_manager::file_descriptor::FileDescriptor;
use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
pub struct FileBucketDirectory {
file_object_hash: String,
remote_cloud_base_directory: String,
full_remote_cloud_file_path: String,
file_name: String,
file_descriptor: Box<dyn FileDescriptor>
}
impl FileBucketDirectory {
pub fn generate_new(file_descriptor: Box<dyn FileDescriptor>) -> Self {
Self::new_from_file_descriptor(file_descriptor)
}
// TODO refactor this out of here
pub fn from_existing_bucket_details(bucket_details:RemoteCloudBucketDetails) -> Self {
let file_descriptor = bucket_details.file_descriptor_from_bucket_details();
let entropy = bucket_details.get_object_hash();
let middle = hashed_directory_path_long_string::hashed_directory_path_long_string(entropy.as_ref());
// gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
// gets you name of the file with suffix and prefix and entropy in the centre
let file_name = format!("{}_{}.{}", bucket_details.get_prefix(), entropy, bucket_details.get_suffix());
// note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
// This is the full path you download from
let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name);
Self {
file_object_hash: entropy.to_string(),
remote_cloud_base_directory: remote_cloud_base_directory,
full_remote_cloud_file_path: full_remote_cloud_file_path,
file_name:file_name,
file_descriptor:file_descriptor
}
}
pub fn new_from_file_descriptor(file_descriptor: Box<dyn FileDescriptor>) -> Self {
let entropy = crockford_entropy_lower(32);
// gets you wiki /a/b/c/d folder structure
let middle = hashed_directory_path_long_string::hashed_directory_path_long_string(entropy.as_ref());
// gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
// gets you name of the file with suffix and prefix and entropy in the centre
let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix());
// note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
// This is the full path you upload to.
let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name);
Self {
file_object_hash: entropy,
remote_cloud_base_directory: remote_cloud_base_directory,
full_remote_cloud_file_path: full_remote_cloud_file_path,
file_name:file_name,
file_descriptor:file_descriptor
}
}
pub fn get_file_object_hash(&self) -> &str {
&self.file_object_hash
}
pub fn get_remote_cloud_base_directory(&self) -> &str {
&self.remote_cloud_base_directory
}
pub fn get_full_remote_cloud_file_path(&self) -> &str {
&self.full_remote_cloud_file_path
}
pub fn to_full_remote_cloud_file_path_pathbuf(&self) -> PathBuf {
PathBuf::from(&self.full_remote_cloud_file_path)
}
pub fn get_file_name(&self) -> &str {
&self.file_name
}
}
@@ -0,0 +1,6 @@
#[derive(Debug, Clone)]
pub struct FileMetaData {
pub file_size_bytes: u64,
pub sha256_checksum: String,
pub mimetype: String
}
@@ -0,0 +1,12 @@
pub mod remote_cloud_file_manager;
pub mod remote_cloud_bucket_details;
pub mod bucket_orchestration;
pub mod file_directory;
pub mod file_meta_data;
pub mod file_descriptor;
pub mod media_descriptor;
pub mod weights_descriptor;
@@ -0,0 +1,58 @@
use super::file_descriptor::FileDescriptor;
use crate::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor, WeightsSDXLDescriptor, self};
pub struct RemoteCloudBucketDetails {
pub object_hash: String,
pub prefix: String,
pub suffix: String,
}
impl RemoteCloudBucketDetails {
pub fn new(object_hash: String, prefix: String, suffix: String) -> Self {
Self {
object_hash: object_hash,
prefix: prefix,
suffix: suffix
}
}
pub fn get_object_hash(&self) -> &str {
&self.object_hash
}
pub fn get_prefix(&self) -> &str {
&self.prefix
}
pub fn get_suffix(&self) -> &str {
&self.suffix
}
pub fn file_descriptor_from_bucket_details(&self) -> Box<dyn FileDescriptor> {
match self.prefix.as_str() {
"loRA" => Box::new(weights_descriptor::WeightsLoRADescriptor {}),
"sd15" => Box::new(weights_descriptor::WeightsSD15Descriptor {}),
"sdxl" => Box::new(weights_descriptor::WeightsSDXLDescriptor {}),
"valle_prompt" => Box::new(weights_descriptor::WeightsVallePromptDescriptor {}),
"rvc" => {
match self.suffix.as_str() {
"safetensors" => Box::new(weights_descriptor::WeightsRVCDescriptor {}),
"index" => Box::new(weights_descriptor::WeightsRVCIndexDescriptor {}),
_ => panic!("Unknown suffix: {}",self.suffix)
}
},
"svc" => Box::new(weights_descriptor::WeightsSVCDescriptor {}),
_ => panic!("Unknown prefix: {}", self.prefix)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
pub fn test() {
}
}
@@ -0,0 +1,196 @@
use crate::remote_file_manager::file_directory::FileBucketDirectory;
use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use super::bucket_orchestration;
use super::file_descriptor::{FileDescriptor};
use super::file_meta_data::FileMetaData;
use errors:: AnyhowResult;
use filesys::file_read_bytes::file_read_bytes;
use filesys::file_size::file_size;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use crate::remote_file_manager::bucket_orchestration::{BucketOrchestrationCore};
struct RemoteCloudFileClient {
bucket_orchestration_client: Box<dyn BucketOrchestrationCore>
}
impl RemoteCloudFileClient {
pub fn new(bucket_orchestration_client: Box<dyn BucketOrchestrationCore>) -> Self {
Self {
bucket_orchestration_client: bucket_orchestration_client
}
}
pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyhowResult<()> {
let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details();
let file_bucket_directory = FileBucketDirectory::from_existing_bucket_details(remote_cloud_bucket_details);
let full_remote_cloud_file_path = file_bucket_directory.get_full_remote_cloud_file_path().to_string();
let is_public = file_descriptor.is_public().clone();
self.bucket_orchestration_client.download_file_to_disk(full_remote_cloud_file_path, to_system_file_path, is_public).await?;
Ok(())
}
// return error or success with meta data.
pub async fn upload_file(&self, file_descriptor:Box<dyn FileDescriptor>, from_system_file_path:&str) -> AnyhowResult<FileMetaData> {
// get file meta data
println!("Reading media file: {:?}", from_system_file_path);
// get meta data
let bytes = file_read_bytes(from_system_file_path)?;
let result = Self::get_file_meta_data(from_system_file_path)?;
let is_public = file_descriptor.is_public();
let directory = FileBucketDirectory::generate_new(
file_descriptor
);
println!("Uploading media file to bucket path: {:?}",directory.get_full_remote_cloud_file_path());
self.bucket_orchestration_client.upload_file_with_content_type_process(
&directory.get_remote_cloud_base_directory(),
bytes.as_ref(),
result.mimetype.as_ref(),
is_public
).await?;
Ok(result)
}
// Retrieve the metadata from the file
fn get_file_meta_data(system_file_path:&str) -> AnyhowResult<FileMetaData> {
let file_size_bytes = file_size(system_file_path.clone())?;
let sha256_checksum = sha256_hash_file(system_file_path.clone())?;
let bytes = file_read_bytes(system_file_path)?;
let mimetype: &str = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream");
Ok(FileMetaData {
file_size_bytes: file_size_bytes,
sha256_checksum: sha256_checksum,
mimetype: mimetype.to_string()
})
}
}
#[cfg(test)]
mod tests {
use async_trait::async_trait;
use env_logger;
use errors::AnyhowResult;
use crate::remote_file_manager::bucket_orchestration::BucketOrchestrationCore;
use crate::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor, WeightsSDXLDescriptor};
struct BucketOrchestrationMock {}
#[async_trait]
impl BucketOrchestrationCore for BucketOrchestrationMock {
async fn download_file_to_disk(
&self,
object_path: String,
filesystem_path: String,
is_public: bool,
) -> AnyhowResult<()>{
println!("Download File to Disk");
println!("{}",object_path);
println!("{}",filesystem_path);
println!("{}",is_public);
println!("Download Done Downloading");
assert_eq!(String::from("/weights/1/2/123/loRA_123.safetensors"),object_path);
assert_eq!(String::from("./file_path_here"),filesystem_path);
assert_eq!(is_public,true);
Ok(())
}
async fn upload_file_with_content_type_process(&self, object_name: &str,
bytes: &[u8],
content_type: &str,
is_public: bool) -> AnyhowResult<()> {
println!("Upload File to Disk");
println!("{}",object_name);
assert_eq!(object_name,String::from("/weights/2/y/q/m/2/2yqm2f1bamh88seyd690h9v24apgezhr/loRA_2yqm2f1bamh88seyd690h9v24apgezhr.safetensors"));
println!("ContentType:{}",content_type);
println!("{}",is_public);
assert_eq!(is_public,true);
println!("Is Done Uploading");
Ok(())
}
}
#[tokio::test]
async fn remote_file_manager_descriptor_test() {
use super::*;
let remote_cloud_bucket_details = RemoteCloudBucketDetails::new("object_hash".to_string(), "loRA".to_string(), "safetensors".to_string());
let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details();
assert_eq!(file_descriptor.get_prefix(), "loRA");
assert_eq!(file_descriptor.get_suffix(), "safetensors");
assert_eq!(file_descriptor.is_public(), true);
}
#[tokio::test]
async fn remote_file_manager_download_existing_file() {
use super::*;
env_logger::init();
let file_path:&str= "./file_path_here";
let remote_cloud_file_manager = RemoteCloudFileClient {
bucket_orchestration_client: Box::new(BucketOrchestrationMock {})
};
println!("begin upload from file_path: {:?}", file_path);
let details = RemoteCloudBucketDetails {
object_hash: String::from("123"),
prefix: String::from("loRA"),
suffix: String::from("safetensors")
};
let result = remote_cloud_file_manager.download_file(details,file_path.to_string()).await;
match result {
Ok(file_meta_data) => {
println!("file_meta_data: {:?}", file_meta_data);
},
Err(e) => {
println!("error: {:?}", e);
}
}
}
#[tokio::test]
async fn remote_file_manager_upload_existing_file() {
use super::*;
env_logger::init();
// Replace with a test file.
let weight_path:&str= "/home/tensor/code/storyteller/storyteller-rust/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs";
let remote_cloud_file_manager = RemoteCloudFileClient {
bucket_orchestration_client: Box::new(BucketOrchestrationMock {})
};
println!("begin upload from weight_path: {:?}", weight_path);
let result = remote_cloud_file_manager.upload_file(Box::new(WeightsLoRADescriptor {}),weight_path).await;
match result {
Ok(file_meta_data) => {
println!("file_meta_data: {:?}", file_meta_data);
},
Err(e) => {
println!("error: {:?}", e);
}
}
}
}
@@ -0,0 +1,172 @@
use super::file_descriptor::FileDescriptor;
pub struct WeightsLoRADescriptor {}
const REMOTE_FILE_DIRECTORY: &str = "/weights";
impl FileDescriptor for WeightsLoRADescriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "safetensors".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, loRA, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "loRA".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
pub struct WeightsSD15Descriptor {}
impl FileDescriptor for WeightsSD15Descriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "safetensors".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "sd15".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
pub struct WeightsSDXLDescriptor {}
impl FileDescriptor for WeightsSDXLDescriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "safetensors".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "sdxl".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
pub struct WeightsSVCDescriptor {}
impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsSVCDescriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "safetensors".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "svc".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
pub struct WeightsRVCDescriptor {}
impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsRVCDescriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "pth".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "rvc".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
pub struct WeightsRVCIndexDescriptor {}
impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsRVCIndexDescriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "index".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "rvc".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
pub struct WeightsVallePromptDescriptor {}
impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsVallePromptDescriptor {
fn remote_directory_path(&self) -> &str {
return REMOTE_FILE_DIRECTORY;
}
// this will be the type of file peroid is handled by the file formatter
// e.g safetensors bin jpg
fn get_suffix(&self)->String {
return "safetensors".to_string();
}
// This will be the prefix of the media type or the weights type.
// name of the weights or the name of the media type
// vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
fn get_prefix(&self)->String {
return "valle_prompt".to_string();
}
// This will be ensure that the right bucket is picked
fn is_public(&self) -> bool {
true
}
}
@@ -0,0 +1 @@
pub mod bucket_orchestration_mock;
@@ -0,0 +1,7 @@
pub mod insert_media_file_from_cli_tool;
pub mod insert_media_file_from_face_animation;
pub mod insert_media_file_from_file_upload;
pub mod insert_media_file_from_rerender;
pub mod insert_media_file_from_tacotron2;
pub mod insert_media_file_from_voice_conversion;
pub mod insert_media_file_from_zero_shot_tts;
@@ -57,7 +57,6 @@ pub struct ListMediaFileForUserArgs<'a> {
}
pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult<MediaFileListPage> {
/// Let's figure out how many results we could have returned total
let count_fields = select_total_count_field();
let mut count_query_builder = query_builder(
@@ -0,0 +1,3 @@
pub mod list_media_files;
pub mod list_media_files_by_tokens;
pub mod list_media_files_for_user;
@@ -1,13 +1,5 @@
pub mod create;
pub mod delete_media_file;
pub mod get_media_file;
pub mod insert_media_file_from_cli_tool;
pub mod insert_media_file_from_face_animation;
pub mod insert_media_file_from_file_upload;
pub mod insert_media_file_from_rerender;
pub mod insert_media_file_from_tacotron2;
pub mod insert_media_file_from_voice_conversion;
pub mod insert_media_file_from_zero_shot_tts;
pub mod list_media_files;
pub mod list_media_files_by_tokens;
pub mod list_media_files_for_user;
pub mod list;
pub mod update_media_file;
@@ -1 +1,2 @@
pub mod create_model_weight_from_voice_conversion_download_job;
pub mod create_model_weight_from_voice_conversion_download_job;
pub mod create_weight;
@@ -0,0 +1,2 @@
pub mod set_model_weight_avatar;
pub mod update_weight;
@@ -0,0 +1,32 @@
use sqlx::MySqlPool;
use errors::AnyhowResult;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
pub struct UpdateArgs<'a> {
pub model_weight_token: &'a ModelWeightToken,
pub maybe_avatar_media_file_token: Option<&'a MediaFileToken>,
//pub update_ip_address: &'a str,
pub mysql_pool: &'a MySqlPool
}
pub async fn set_model_weight_avatar(args: UpdateArgs<'_>) -> AnyhowResult<()>{
let transaction = args.mysql_pool.begin().await?;
let query_result = sqlx::query!(
r#"
UPDATE model_weights
SET
maybe_avatar_media_file_token = ?
WHERE token = ?
LIMIT 1
"#,
args.maybe_avatar_media_file_token,
args.model_weight_token,
).execute(args.mysql_pool).await;
transaction.commit().await?;
Ok(())
}
@@ -14,7 +14,7 @@ use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
// 'weights_type: enums::by_table::model_weights::weights_types::WeightsType' use this to map
// Retrieved Model Weight can be constrained to the fields that are needed
pub struct RetrivedModelWeight {
pub struct RetrievedModelWeight {
pub token: ModelWeightToken,
pub title: String,
pub weights_type: WeightsType,
@@ -22,9 +22,15 @@ pub struct RetrivedModelWeight {
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_username: String,
pub creator_display_name: String,
pub creator_gravatar_hash: String,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
pub original_download_url: Option<String>,
pub original_filename: Option<String>,
@@ -33,6 +39,11 @@ pub struct RetrivedModelWeight {
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub maybe_avatar_public_bucket_hash: Option<String>,
pub maybe_avatar_public_bucket_prefix: Option<String>,
pub maybe_avatar_public_bucket_extension: Option<String>,
pub cached_user_ratings_negative_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_total_count: u32,
@@ -49,7 +60,7 @@ pub async fn get_weight_by_token(
weight_token: &ModelWeightToken,
can_see_deleted: bool,
mysql_pool: &MySqlPool
) -> AnyhowResult<Option<RetrivedModelWeight>> {
) -> AnyhowResult<Option<RetrievedModelWeight>> {
let mut connection = mysql_pool.acquire().await?;
get_weights_by_token_with_connection(weight_token, can_see_deleted, &mut connection).await
}
@@ -58,7 +69,7 @@ pub async fn get_weights_by_token_with_connection(
weight_token: &ModelWeightToken,
can_see_deleted: bool,
mysql_connection: &mut PoolConnection<MySql>
) -> AnyhowResult<Option<RetrivedModelWeight>> {
) -> AnyhowResult<Option<RetrievedModelWeight>> {
let maybe_result = if can_see_deleted {
select_include_deleted(weight_token, mysql_connection).await
} else {
@@ -79,7 +90,7 @@ pub async fn get_weights_by_token_with_connection(
// unwrap the result
Ok(
Some(RetrivedModelWeight {
Some(RetrievedModelWeight {
token: record.token,
title: record.title,
weights_type: record.weights_type,
@@ -88,6 +99,9 @@ pub async fn get_weights_by_token_with_connection(
description_markdown: record.description_markdown,
description_rendered_html: record.description_rendered_html,
creator_user_token: record.creator_user_token,
creator_username: record.creator_username,
creator_display_name: record.creator_display_name,
creator_gravatar_hash: record.creator_gravatar_hash,
creator_ip_address: record.creator_ip_address,
creator_set_visibility: record.creator_set_visibility,
maybe_last_update_user_token: record.maybe_last_update_user_token,
@@ -98,6 +112,9 @@ pub async fn get_weights_by_token_with_connection(
public_bucket_hash: record.public_bucket_hash,
maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
maybe_public_bucket_extension: record.maybe_public_bucket_extension,
maybe_avatar_public_bucket_hash: record.maybe_avatar_public_bucket_hash,
maybe_avatar_public_bucket_prefix: record.maybe_avatar_public_bucket_prefix,
maybe_avatar_public_bucket_extension: record.maybe_avatar_public_bucket_extension,
cached_user_ratings_negative_count: record.cached_user_ratings_negative_count,
cached_user_ratings_positive_count: record.cached_user_ratings_positive_count,
cached_user_ratings_total_count: record.cached_user_ratings_total_count,
@@ -128,7 +145,12 @@ async fn select_include_deleted(
wt.maybe_thumbnail_token,
wt.description_markdown,
wt.description_rendered_html,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash AS creator_gravatar_hash,
wt.creator_ip_address,
wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
@@ -139,6 +161,11 @@ async fn select_include_deleted(
wt.public_bucket_hash,
wt.maybe_public_bucket_prefix,
wt.maybe_public_bucket_extension,
avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,
avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,
avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,
wt.cached_user_ratings_negative_count,
wt.cached_user_ratings_positive_count,
wt.cached_user_ratings_total_count,
@@ -150,6 +177,10 @@ async fn select_include_deleted(
wt.user_deleted_at,
wt.mod_deleted_at
FROM model_weights as wt
JOIN users
ON users.token = wt.creator_user_token
LEFT OUTER JOIN media_files as avatar
ON avatar.token = wt.maybe_avatar_media_file_token
WHERE
wt.token = ?
"#,
@@ -176,9 +207,14 @@ async fn select_without_deleted(
wt.maybe_thumbnail_token,
wt.description_markdown,
wt.description_rendered_html,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
wt.creator_ip_address,
wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash AS creator_gravatar_hash,
wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
wt.original_download_url,
wt.original_filename,
@@ -187,6 +223,11 @@ async fn select_without_deleted(
wt.public_bucket_hash,
wt.maybe_public_bucket_prefix,
wt.maybe_public_bucket_extension,
avatar.public_bucket_directory_hash as maybe_avatar_public_bucket_hash,
avatar.maybe_public_bucket_prefix as maybe_avatar_public_bucket_prefix,
avatar.maybe_public_bucket_extension as maybe_avatar_public_bucket_extension,
wt.cached_user_ratings_negative_count,
wt.cached_user_ratings_positive_count,
wt.cached_user_ratings_total_count,
@@ -198,6 +239,10 @@ async fn select_without_deleted(
wt.user_deleted_at,
wt.mod_deleted_at
FROM model_weights as wt
JOIN users
ON users.token = wt.creator_user_token
LEFT OUTER JOIN media_files as avatar
ON avatar.token = wt.maybe_avatar_media_file_token
WHERE
wt.token = ?
AND wt.user_deleted_at IS NULL
@@ -218,7 +263,12 @@ pub struct RawWeight {
pub maybe_thumbnail_token: Option<String>,
pub description_markdown: String,
pub description_rendered_html: String,
pub creator_user_token: UserToken,
pub creator_username: String,
pub creator_display_name: String,
pub creator_gravatar_hash: String,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_last_update_user_token: Option<UserToken>,
@@ -229,6 +279,11 @@ pub struct RawWeight {
pub public_bucket_hash: String,
pub maybe_public_bucket_prefix: Option<String>,
pub maybe_public_bucket_extension: Option<String>,
pub maybe_avatar_public_bucket_hash: Option<String>,
pub maybe_avatar_public_bucket_prefix: Option<String>,
pub maybe_avatar_public_bucket_extension: Option<String>,
pub cached_user_ratings_negative_count: u32,
pub cached_user_ratings_positive_count: u32,
pub cached_user_ratings_total_count: u32,
@@ -1,15 +1,27 @@
use chrono::{DateTime, Utc};
use sqlx::{Acquire, MySqlConnection, MySqlPool};
use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
use sqlx::mysql::MySqlRow;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use enums::traits::mysql_from_row::MySqlFromRow;
use errors::AnyhowResult;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
pub struct WeightsForUserListPage {
pub records: Vec<WeightsJoinUserRecord>,
pub sort_ascending: bool,
pub current_page: usize,
pub total_page_count: usize,
}
#[derive(Serialize)]
pub struct WeightsJoinUserRecord {
pub token: ModelWeightToken,
@@ -59,47 +71,83 @@ pub struct WeightsJoinUserRecord {
pub creator_email_gravatar_hash: String,
}
pub async fn list_weights_by_creator_username(
mysql_pool: &MySqlPool,
creator_username: &str,
can_see_deleted: bool
) -> AnyhowResult<Vec<WeightsJoinUserRecord>> {
let mut connection = mysql_pool.acquire().await?;
let raw_weights: Vec<RawWeightJoinUser> = get_raw_weights_by_creator_username(&mut connection, creator_username, can_see_deleted).await?;
let weights_records: Vec<WeightsJoinUserRecord> = map_to_weights(raw_weights).await;
Ok(weights_records)
pub struct ListWeightsForUserArgs<'a> {
pub creator_username: &'a str,
pub page_size: usize,
pub page_index: usize,
pub sort_ascending: bool,
pub can_see_deleted: bool,
pub mysql_pool: &'a MySqlPool,
}
async fn get_raw_weights_by_creator_username(
connection: &mut MySqlConnection,
creator_username: &str,
can_see_deleted: bool
) -> AnyhowResult<Vec<RawWeightJoinUser>> {
pub async fn list_weights_by_creator_username(args: ListWeightsForUserArgs<'_>) -> AnyhowResult<WeightsForUserListPage> {
let count_fields = select_total_count_field();
let mut count_query_builder = query_builder(
args.creator_username,
false,
0,
0,
args.sort_ascending,
count_fields.as_str(),
args.can_see_deleted,
);
let connection = connection.acquire().await?;
if can_see_deleted {
let raw_weights: Vec<RawWeightJoinUser> = sqlx::query_as!(
RawWeightJoinUser,
r#"
SELECT
mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
let row_count_query = count_query_builder.build_query_scalar::<i64>();
let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?;
/// Now fetch the actual results with all the fields
let result_fields = select_result_fields();
let mut query = query_builder(
args.creator_username,
true,
args.page_index,
args.page_size,
args.sort_ascending,
result_fields.as_str(),
args.can_see_deleted,
);
let query = query.build_query_as::<RawWeightJoinUser>();
let results = query.fetch_all(args.mysql_pool).await?;
let number_of_pages = (row_count_result / args.page_size as i64) as usize;
let weights_records: Vec<WeightsJoinUserRecord> = map_to_weights(results).await;
Ok(WeightsForUserListPage {
records: weights_records,
sort_ascending: args.sort_ascending,
current_page: args.page_index,
total_page_count: number_of_pages,
})
}
fn select_total_count_field() -> String {
r#"
COUNT(mw.id) AS total_count
"#
.to_string()
}
fn select_result_fields() -> String {
r#"
mw.token,
mw.title,
mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,
mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,
mw.weights_type,
mw.weights_category,
mw.maybe_thumbnail_token,
mw.description_markdown,
mw.description_rendered_html,
users.token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash as creator_email_gravatar_hash,
u.token as creator_user_token,
u.username as creator_username,
u.display_name as creator_display_name,
u.email_gravatar_hash as creator_email_gravatar_hash,
mw.creator_ip_address,
mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
mw.creator_set_visibility,
mw.maybe_last_update_user_token,
mw.original_download_url,
mw.original_filename,
mw.file_size_bytes,
@@ -117,65 +165,53 @@ async fn get_raw_weights_by_creator_username(
mw.updated_at,
mw.user_deleted_at,
mw.mod_deleted_at
FROM model_weights as mw
JOIN users
ON users.token = mw.creator_user_token
WHERE
users.username = ?
"#,
creator_username).fetch_all(connection)
.await?;
Ok(raw_weights)
} else {
let raw_weights: Vec<RawWeightJoinUser> = sqlx::query_as!(
RawWeightJoinUser,
r#"
SELECT
mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
mw.title,
mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,
mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,
mw.maybe_thumbnail_token,
mw.description_markdown,
mw.description_rendered_html,
users.token as `creator_user_token: tokens::tokens::users::UserToken`,
users.username as creator_username,
users.display_name as creator_display_name,
users.email_gravatar_hash as creator_email_gravatar_hash,
mw.creator_ip_address,
mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
mw.original_download_url,
mw.original_filename,
mw.file_size_bytes,
mw.file_checksum_sha2,
mw.public_bucket_hash,
mw.maybe_public_bucket_prefix,
mw.maybe_public_bucket_extension,
mw.cached_user_ratings_negative_count,
mw.cached_user_ratings_positive_count,
mw.cached_user_ratings_total_count,
mw.maybe_cached_user_ratings_ratio,
mw.cached_user_ratings_last_updated_at,
mw.version,
mw.created_at,
mw.updated_at,
mw.user_deleted_at,
mw.mod_deleted_at
FROM model_weights as mw
JOIN users
ON users.token = mw.creator_user_token
WHERE
users.username = ?
AND mw.user_deleted_at IS NULL
AND mw.mod_deleted_at IS NULL
"#,
creator_username).fetch_all(connection).await?;
"#.to_string()
}
Ok(raw_weights)
fn query_builder<'a>(
username: &'a str,
enforce_limits: bool,
page_index: usize,
page_size: usize,
sort_ascending: bool,
select_fields: &'a str,
can_see_deleted: bool,
) -> QueryBuilder<'a, MySql> {
// NB: Query cannot be statically checked by sqlx
let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
format!(r#"
SELECT
{select_fields}
FROM model_weights as mw
JOIN users as u
ON u.token = mw.creator_user_token
"#
));
if !can_see_deleted {
query_builder.push(" WHERE mw.user_deleted_at IS NULL AND mw.mod_deleted_at IS NULL ");
}
query_builder.push(" AND u.username = ");
query_builder.push_bind(username);
if sort_ascending {
query_builder.push(" ORDER BY mw.created_at ASC ");
} else {
query_builder.push(" ORDER BY mw.created_at DESC ");
}
if enforce_limits {
let offset = page_index * page_size;
query_builder.push(format!(" LIMIT {page_size} OFFSET {offset} "));
}
query_builder
}
async fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsJoinUserRecord> {
let weights: Vec<WeightsJoinUserRecord> = dataset
.into_iter()
@@ -224,7 +260,6 @@ async fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsJoinUserRe
}
#[derive(Serialize)]
pub struct RawWeightJoinUser {
pub token: ModelWeightToken,
@@ -272,3 +307,41 @@ async fn map_to_weights(dataset:Vec<RawWeightJoinUser>) -> Vec<WeightsJoinUserRe
pub creator_display_name: String,
pub creator_email_gravatar_hash: String,
}
impl FromRow<'_, MySqlRow> for RawWeightJoinUser {
fn from_row(row: &MySqlRow) -> std::result::Result<Self, sqlx::Error> {
Ok(Self {
token: row.try_get("token")?,
weights_type: row.try_get("weights_type")?,
weights_category: row.try_get("weights_category")?,
title: row.try_get("title")?,
maybe_thumbnail_token: row.try_get("maybe_thumbnail_token")?,
description_markdown: row.try_get("description_markdown")?,
description_rendered_html: row.try_get("description_rendered_html")?,
creator_user_token: row.try_get("creator_user_token")?,
creator_ip_address: row.try_get("creator_ip_address")?,
creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
maybe_last_update_user_token: row.try_get("maybe_last_update_user_token")?,
original_download_url: row.try_get("original_download_url")?,
original_filename: row.try_get("original_filename")?,
file_size_bytes: row.try_get("file_size_bytes")?,
file_checksum_sha2: row.try_get("file_checksum_sha2")?,
public_bucket_hash: row.try_get("public_bucket_hash")?,
maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
cached_user_ratings_total_count: row.try_get("cached_user_ratings_total_count")?,
cached_user_ratings_positive_count: row.try_get("cached_user_ratings_positive_count")?,
cached_user_ratings_negative_count: row.try_get("cached_user_ratings_negative_count")?,
maybe_cached_user_ratings_ratio: row.try_get("maybe_cached_user_ratings_ratio")?,
cached_user_ratings_last_updated_at: row.try_get("cached_user_ratings_last_updated_at")?,
version: row.try_get("version")?,
created_at: row.try_get("created_at")?,
updated_at: row.try_get("updated_at")?,
user_deleted_at: row.try_get("user_deleted_at")?,
mod_deleted_at: row.try_get("mod_deleted_at")?,
creator_username: row.try_get("creator_username")?,
creator_display_name: row.try_get("creator_display_name")?,
creator_email_gravatar_hash: row.try_get("creator_email_gravatar_hash")?,
})
}
}
@@ -2,20 +2,18 @@
#![forbid(unused_mut)]
#![forbid(unused_variables)]
use chrono::{ DateTime, Utc };
use chrono::{DateTime, Utc};
use sqlx::MySqlPool;
use config::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::users::UserToken;
use tokens::tokens::model_weights::ModelWeightToken;
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
};
use tokens::tokens::users::UserToken;
#[derive(Serialize)]
pub struct WeightsPage {
@@ -466,12 +464,13 @@ struct RawWeightJoinUser {
#[cfg(test)]
mod tests {
use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder;
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use crate::queries::model_weights::list::list_weights_query_builder::ListWeightsQueryBuilder;
#[test]
fn predicates_without_scoping() {
let query_builder = ListWeightsQueryBuilder::new();
@@ -1 +1,4 @@
pub mod list_model_weights_for_voice_conversion;
pub mod list_model_weights_for_voice_conversion;
pub mod list_weights_by_tokens;
pub mod list_weights_by_user;
pub mod list_weights_query_builder;
@@ -1,13 +1,9 @@
pub mod create;
pub mod create_weight;
pub mod delete_weights;
pub mod edit;
pub mod get_weight;
pub mod inference;
pub mod list;
pub mod list_weights_by_tokens;
pub mod list_weights_by_user;
pub mod list_weights_query_builder;
pub mod migration;
pub mod model_weight_info_lite;
pub mod tests;
pub mod update_weight;
@@ -3,8 +3,8 @@ mod tests {
use anyhow::Ok;
use rand::Rng;
use serial_test::serial;
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use sqlx::MySqlPool;
use tokio;
use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
@@ -17,8 +17,8 @@ mod tests {
use enums::common::visibility::Visibility;
use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
use crate::queries::model_weights::create_weight::create_weight;
use crate::queries::model_weights::create_weight::CreateModelWeightsArgs;
use crate::queries::model_weights::create::create_weight::create_weight;
use crate::queries::model_weights::create::create_weight::CreateModelWeightsArgs;
use crate::queries::model_weights::delete_weights::{
delete_weights_as_mod,
delete_weights_as_user,
@@ -26,8 +26,8 @@ mod tests {
undelete_weights_as_user,
};
use crate::queries::model_weights::get_weight::get_weight_by_token;
use crate::queries::model_weights::list_weights_by_user::list_weights_by_creator_username;
use crate::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder;
use crate::queries::model_weights::list::list_weights_by_user::{list_weights_by_creator_username, ListWeightsForUserArgs};
use crate::queries::model_weights::list::list_weights_query_builder::ListWeightsQueryBuilder;
use crate::queries::users::user::get_user_token_by_username::get_user_token_by_username;
async fn setup() -> sqlx::Pool<sqlx::MySql> {
@@ -290,16 +290,19 @@ mod tests {
&creator_username
).await?;
let weights_by_username = list_weights_by_creator_username(
&pool,
&creator_username,
can_see_deleted
).await?;
for weight in weights_by_username.iter() {
let weights_by_username = list_weights_by_creator_username(ListWeightsForUserArgs {
creator_username: &creator_username,
page_size: 0,
page_index: 0,
sort_ascending: false,
can_see_deleted,
mysql_pool: &pool,
}).await?;
for weight in weights_by_username.records.iter() {
// print weight
println!("weight: {:?}", weight.creator_username);
}
assert_eq!(weights_by_username.len(), 5);
assert_eq!(weights_by_username.records.len(), 5);
Ok(())
}
@@ -9,7 +9,6 @@ use filesys::file_exists::file_exists;
use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelError, VcModelType};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord};
use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInferenceError;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job_dependencies::JobDependencies;
@@ -13,7 +13,7 @@ use filesys::safe_delete_temp_file::safe_delete_temp_file;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs};
use tokens::tokens::users::UserToken;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -13,7 +13,7 @@ use hashing::sha256::sha256_hash_string::sha256_hash_string;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::column_types::vocoder_type::VocoderType;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_tacotron2::{insert_media_file_from_tacotron2, InsertMediaFileArgs};
use mysql_queries::queries::tts::tts_results::insert_tts_result::insert_tts_result;
use mysql_queries::queries::tts::tts_results::insert_tts_result::JobType;
@@ -13,8 +13,8 @@ use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::file_size::file_size;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mysql_queries::queries::media_files::insert_media_file_from_zero_shot_tts::insert_media_file_from_zero_shot;
use mysql_queries::queries::media_files::insert_media_file_from_zero_shot_tts::InsertArgs;
use mysql_queries::queries::media_files::create::insert_media_file_from_zero_shot_tts::insert_media_file_from_zero_shot;
use mysql_queries::queries::media_files::create::insert_media_file_from_zero_shot_tts::InsertArgs;
use mysql_queries::queries::voice_designer::voices::get_voice::{get_voice_by_token, ZsVoice};
use crate::job::job_loop::job_success_result::JobSuccessResult;
@@ -19,7 +19,7 @@ use migration::voice_conversion::query_vc_model_for_migration::VcModel;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_files::create::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::model_weights::ModelWeightToken;
@@ -18,7 +18,7 @@ use migration::voice_conversion::query_vc_model_for_migration::VcModel;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_files::create::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::model_weights::ModelWeightToken;
@@ -7,7 +7,7 @@ use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::model_weights::get_weight::RetrivedModelWeight;
use mysql_queries::queries::model_weights::get_weight::RetrievedModelWeight;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
@@ -18,7 +18,7 @@ pub struct ModelFile {
}
pub async fn download_model_file(
model_record: &Option<RetrivedModelWeight>,
model_record: &Option<RetrievedModelWeight>,
public_bucket_client: &BucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
@@ -4,8 +4,8 @@ use std::time::Instant;
use anyhow::{anyhow, Error};
use log::{error, info, warn};
use serde::{Deserialize, Serialize};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::file_size::file_size;
@@ -15,7 +15,7 @@ use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs};
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -18,11 +18,21 @@ use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::*;
use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::*;
use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::*;
use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::*;
use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::*;
use crate::http_server::endpoints::user_ratings::get_user_rating_handler::*;
use crate::http_server::endpoints::user_ratings::set_user_rating_handler::*;
use tokens::tokens::generic_inference_jobs::*;
use tokens::tokens::model_weights::*;
use tokens::tokens::users::*;
use tokens::tokens::media_files::*;
use tokens::tokens::user_bookmarks::*;
use enums::common::visibility::Visibility;
use chrono::{DateTime, Utc};
use utoipa::OpenApi;
@@ -36,7 +46,11 @@ use utoipa::OpenApi;
crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler,
crate::http_server::endpoints::media_files::list_featured_media_files::list_featured_media_files_handler,
crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler,
crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler
crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler,
crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler::create_user_bookmark_handler,
crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler,
crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler,
crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler,
),components(schemas(
// Tokens
UserToken,Visibility,
@@ -60,6 +74,11 @@ use utoipa::OpenApi;
ListMediaFilesSuccessResponse, ListMediaFilesError, MediaFileListItem, ListMediaFilesQueryParams,
MediaFileSocialMetaLight,
ListMediaFilesForUserSuccessResponse,ListMediaFilesForUserQueryParams,ListMediaFilesForUserError,ListMediaFilesForUserPathInfo,
MediaFileForUserListItem, MediaFileForUserListItem
MediaFileForUserListItem, MediaFileForUserListItem,
UserBookmarkToken,UserBookmarkDetailsForUserList,UserBookmarkEntityType,
CreateUserBookmarkRequest,CreateUserBookmarkError,CreateUserBookmarkSuccessResponse,
DeleteUserBookmarkPathInfo,DeleteUserBookmarkError,DeleteUserBookmarkRequest,
ListUserBookmarksPathInfo,ListUserBookmarksForUserError,ListUserBookmarksForUserSuccessResponse,UserBookmarkListItem,
ListUserBookmarksForEntityPathInfo,ListUserBookmarksForEntityError,ListUserBookmarksForEntitySuccessResponse,UserBookmarkForEntityListItem,
)))]
pub struct ApiDoc;
@@ -7,20 +7,11 @@ use chrono::{DateTime, Utc};
use log::{debug, error, warn};
use r2d2_redis::redis::Commands;
use utoipa::ToSchema;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use mysql_queries::queries::media_files::list_media_files_by_tokens::list_media_files_by_tokens;
use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_type::MediaFileType;
use mysql_queries::queries::media_files::list::list_media_files_by_tokens::list_media_files_by_tokens;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
use crate::server_state::ServerState;
@@ -3,21 +3,21 @@ use std::sync::Arc;
use actix_web::{HttpMessage, HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Path, Query};
use actix_web::web::Query;
use chrono::{DateTime, Utc};
use log::{info, warn};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use utoipa::{IntoParams, path, ToSchema};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use mysql_queries::queries::media_files::list_media_files::{list_media_files, ListMediaFilesArgs, ViewAs};
use mysql_queries::queries::media_files::list::list_media_files::{list_media_files, ListMediaFilesArgs, ViewAs};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::common_responses::media_file_social_meta_lite::MediaFileSocialMetaLight;
use crate::http_server::common_responses::media_file_social_meta_lite::MediaFileSocialMetaLight;
use crate::http_server::common_responses::pagination_cursors::PaginationCursors;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
@@ -5,16 +5,16 @@ use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Path, Query};
use chrono::{DateTime, Utc};
use log::{info, warn};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use utoipa::{IntoParams, path, ToSchema};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use mysql_queries::queries::media_files::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs};
use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs, ViewAs};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::common_responses::pagination_page::PaginationPage;
@@ -128,9 +128,7 @@ pub async fn list_media_files_for_user_handler(
// TODO(bt,2023-12-04): Enforce real maximums and defaults
let limit = query.page_size.unwrap_or(25);
let sort_ascending = query.sort_ascending.unwrap_or(false);
let page_size = query.page_size.unwrap_or_else(|| 25);
let page_index = query.page_index.unwrap_or_else(|| 0);
@@ -12,7 +12,7 @@ use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
use mysql_queries::queries::media_files::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use mysql_queries::queries::media_files::create::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upload::drain_multipart_request::{drain_multipart_request, MediaFileUploadSource};
@@ -10,6 +10,7 @@ use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use log::warn;
use utoipa::ToSchema;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use mysql_queries::queries::user_bookmarks::create_user_bookmark::{create_user_bookmark, CreateUserBookmarkArgs};
@@ -28,19 +29,19 @@ use tokens::tokens::zs_voices::ZsVoiceToken;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct CreateUserBookmarkRequest {
entity_token: String,
entity_type: UserBookmarkEntityType,
}
#[derive(Serialize)]
#[derive(Serialize, ToSchema)]
pub struct CreateUserBookmarkSuccessResponse {
pub success: bool,
pub user_bookmark_token: UserBookmarkToken,
}
#[derive(Debug)]
#[derive(Debug, ToSchema)]
pub enum CreateUserBookmarkError {
BadInput(String),
NotAuthorized,
@@ -74,6 +75,15 @@ impl fmt::Display for CreateUserBookmarkError {
}
}
#[utoipa::path(
post,
path = "/v1/user_bookmarks/create",
request_body = CreateUserBookmarkRequest,
responses(
(status = 200, body = CreateUserBookmarkSuccessResponse),
(status = 400, body = CreateUserBookmarkError),
)
)]
pub async fn create_user_bookmark_handler(
http_request: HttpRequest,
request: web::Json<CreateUserBookmarkRequest>,
@@ -11,6 +11,7 @@ use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Path;
use log::warn;
use utoipa::ToSchema;
use mysql_queries::queries::user_bookmarks::delete_user_bookmark::delete_user_bookmark;
use tokens::tokens::user_bookmarks::UserBookmarkToken;
@@ -20,18 +21,18 @@ use crate::http_server::web_utils::response_success_helpers::simple_json_success
use crate::server_state::ServerState;
/// For the URL PathInfo
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct DeleteUserBookmarkPathInfo {
user_bookmark_token: UserBookmarkToken,
}
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct DeleteUserBookmarkRequest {
/// NB: this is only to disambiguate when a user is both a mod and an author.
as_mod: Option<bool>,
}
#[derive(Debug)]
#[derive(Debug, ToSchema)]
pub enum DeleteUserBookmarkError {
BadInput(String),
NotAuthorized,
@@ -68,6 +69,20 @@ impl fmt::Display for DeleteUserBookmarkError {
}
}
#[utoipa::path(
delete,
path = "/v1/user_bookmarks/delete/{user_bookmark_token}",
params(
("user_bookmark_token", description = "UserBookmarkToken"),
),
responses(
(status = 200, description = "Delete User Bookmark", body = SimpleGenericJsonSuccess),
(status = 400, description = "Bad input", body = DeleteUserBookmarkError),
(status = 401, description = "Not authorized", body = DeleteUserBookmarkError),
(status = 404, description = "Not found", body = DeleteUserBookmarkError),
(status = 500, description = "Server error", body = DeleteUserBookmarkError),
),
)]
pub async fn delete_user_bookmark_handler(
http_request: HttpRequest,
path: Path<DeleteUserBookmarkPathInfo>,
@@ -12,6 +12,7 @@ use actix_web::http::StatusCode;
use actix_web::web::Path;
use chrono::{DateTime, Utc};
use log::warn;
use utoipa::ToSchema;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use mysql_queries::queries::user_bookmarks::user_bookmark_entity_token::UserBookmarkEntityToken;
@@ -23,20 +24,20 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
/// For the URL PathInfo
#[derive(Deserialize)]
pub struct ListUserBookmarkPathInfo {
#[derive(Deserialize, ToSchema)]
pub struct ListUserBookmarksForEntityPathInfo {
entity_type: UserBookmarkEntityType,
entity_token: String,
}
#[derive(Serialize)]
pub struct ListUserBookmarksSuccessResponse {
#[derive(Serialize, ToSchema)]
pub struct ListUserBookmarksForEntitySuccessResponse {
pub success: bool,
pub user_bookmarks: Vec<UserBookmark>,
pub user_bookmarks: Vec<UserBookmarkForEntityListItem>,
}
#[derive(Serialize)]
pub struct UserBookmark {
#[derive(Serialize, ToSchema)]
pub struct UserBookmarkForEntityListItem {
pub token: UserBookmarkToken,
pub user: UserDetailsLight,
@@ -45,21 +46,21 @@ pub struct UserBookmark {
pub updated_at: DateTime<Utc>,
}
#[derive(Debug)]
pub enum ListUserBookmarksError {
#[derive(Debug, ToSchema)]
pub enum ListUserBookmarksForEntityError {
ServerError,
}
impl ResponseError for ListUserBookmarksError {
impl ResponseError for ListUserBookmarksForEntityError {
fn status_code(&self) -> StatusCode {
match *self {
ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
ListUserBookmarksForEntityError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
ListUserBookmarksError::ServerError => "server error".to_string(),
ListUserBookmarksForEntityError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
@@ -67,17 +68,30 @@ impl ResponseError for ListUserBookmarksError {
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for ListUserBookmarksError {
impl fmt::Display for ListUserBookmarksForEntityError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[utoipa::path(
get,
path = "/v1/user_bookmarks/list/{entity_type}/{entity_token}",
params(
("entity_type", description="The type of entity to list bookmarks for"),
("entity_token", description="The token of the entity to list bookmarks for"),
),
responses(
(status = 200, body = ListUserBookmarksForEntitySuccessResponse),
(status = 500, body = ListUserBookmarksForEntityError),
),
)]
pub async fn list_user_bookmarks_for_entity_handler(
_http_request: HttpRequest,
path: Path<ListUserBookmarkPathInfo>,
path: Path<ListUserBookmarksForEntityPathInfo>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ListUserBookmarksError>
) -> Result<HttpResponse, ListUserBookmarksForEntityError>
{
let entity_token = UserBookmarkEntityToken::from_entity_type_and_token(
path.entity_type, &path.entity_token);
@@ -91,14 +105,14 @@ pub async fn list_user_bookmarks_for_entity_handler(
Ok(results) => results,
Err(e) => {
warn!("Query error: {:?}", e);
return Err(ListUserBookmarksError::ServerError);
return Err(ListUserBookmarksForEntityError::ServerError);
}
};
let response = ListUserBookmarksSuccessResponse {
let response = ListUserBookmarksForEntitySuccessResponse {
success: true,
user_bookmarks: user_bookmarks.into_iter()
.map(|user_bookmark| UserBookmark {
.map(|user_bookmark| UserBookmarkForEntityListItem {
token: user_bookmark.token,
user: UserDetailsLight {
user_token: user_bookmark.user_token.clone(),
@@ -114,7 +128,7 @@ pub async fn list_user_bookmarks_for_entity_handler(
};
let body = serde_json::to_string(&response)
.map_err(|_e| ListUserBookmarksError::ServerError)?;
.map_err(|_e| ListUserBookmarksForEntityError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -12,6 +12,7 @@ use actix_web::http::StatusCode;
use actix_web::web::{Path, Query};
use chrono::{DateTime, Utc};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use mysql_queries::queries::user_bookmarks::list_user_bookmarks::{list_user_bookmarks, list_user_user_bookmarks_by_entity_type};
@@ -20,34 +21,34 @@ use tokens::tokens::user_bookmarks::UserBookmarkToken;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema)]
pub struct ListUserBookmarksPathInfo {
username: String,
}
#[derive(Deserialize)]
#[derive(Deserialize, ToSchema, IntoParams)]
pub struct ListUserBookmarksQueryData {
maybe_scoped_entity_type: Option<UserBookmarkEntityType>,
}
#[derive(Serialize)]
pub struct ListUserBookmarksSuccessResponse {
#[derive(Serialize, ToSchema)]
pub struct ListUserBookmarksForUserSuccessResponse {
pub success: bool,
pub user_bookmarks: Vec<UserBookmark>,
pub user_bookmarks: Vec<UserBookmarkListItem>,
}
#[derive(Serialize)]
pub struct UserBookmark {
#[derive(Serialize, ToSchema)]
pub struct UserBookmarkListItem {
pub token: UserBookmarkToken,
pub details: UserBookmarkDetails,
pub details: UserBookmarkDetailsForUserList,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Serialize)]
pub struct UserBookmarkDetails {
#[derive(Serialize, ToSchema)]
pub struct UserBookmarkDetailsForUserList {
// TODO: This needs titles or some other summary metadata.
pub entity_type: UserBookmarkEntityType,
pub entity_token: String,
@@ -59,21 +60,21 @@ pub struct UserBookmarkDetails {
pub maybe_thumbnail_url: Option<String>,
}
#[derive(Debug)]
pub enum ListUserBookmarksError {
#[derive(Debug, ToSchema)]
pub enum ListUserBookmarksForUserError {
ServerError,
}
impl ResponseError for ListUserBookmarksError {
impl ResponseError for ListUserBookmarksForUserError {
fn status_code(&self) -> StatusCode {
match *self {
ListUserBookmarksError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
ListUserBookmarksForUserError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
ListUserBookmarksError::ServerError => "server error".to_string(),
ListUserBookmarksForUserError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
@@ -81,18 +82,30 @@ impl ResponseError for ListUserBookmarksError {
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for ListUserBookmarksError {
impl fmt::Display for ListUserBookmarksForUserError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[utoipa::path(
get,
path = "/v1/user_bookmarks/list/user/{username}",
params(
("username", description = "The username of the user whose bookmarks to list."),
ListUserBookmarksQueryData
),
responses(
(status = 200, description = "List User Bookmarks", body = ListUserBookmarksForUserSuccessResponse),
(status = 500, description = "Server error", body = ListUserBookmarksForUserError),
),
)]
pub async fn list_user_bookmarks_for_user_handler(
_http_request: HttpRequest,
path: Path<ListUserBookmarksPathInfo>,
query: Query<ListUserBookmarksQueryData>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ListUserBookmarksError>
) -> Result<HttpResponse, ListUserBookmarksForUserError>
{
let query_results = match query.maybe_scoped_entity_type {
@@ -106,16 +119,16 @@ pub async fn list_user_bookmarks_for_user_handler(
Ok(results) => results,
Err(e) => {
warn!("Query error: {:?}", e);
return Err(ListUserBookmarksError::ServerError);
return Err(ListUserBookmarksForUserError::ServerError);
}
};
let response = ListUserBookmarksSuccessResponse {
let response = ListUserBookmarksForUserSuccessResponse {
success: true,
user_bookmarks: user_bookmarks.into_iter()
.map(|user_bookmark| UserBookmark {
.map(|user_bookmark| UserBookmarkListItem {
token: user_bookmark.token,
details: UserBookmarkDetails {
details: UserBookmarkDetailsForUserList {
entity_type: user_bookmark.entity_type,
entity_token: user_bookmark.entity_token,
maybe_summary_text: user_bookmark.maybe_entity_descriptive_text,
@@ -130,7 +143,7 @@ pub async fn list_user_bookmarks_for_user_handler(
};
let body = serde_json::to_string(&response)
.map_err(|_e| ListUserBookmarksError::ServerError)?;
.map_err(|_e| ListUserBookmarksForUserError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -1,25 +1,24 @@
use std::fmt;
use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Path;
use chrono::{DateTime, Utc};
use log::warn;
use utoipa::ToSchema;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::model_weights::weights_category::WeightsCategory;
use enums::by_table::model_weights::weights_types::WeightsType;
use enums::common::visibility::Visibility;
use tokens::tokens::users::UserToken;
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use tokens::tokens::model_weights::ModelWeightToken;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
use crate::server_state::ServerState;
use tokens::tokens::model_weights::ModelWeightToken;
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use enums::by_table::model_weights::weights_types::WeightsType;
use enums::by_table::model_weights::weights_category::WeightsCategory;
use utoipa::ToSchema;
#[derive(Serialize, Clone, ToSchema)]
pub struct GetWeightResponse {
success: bool,
@@ -30,11 +29,16 @@ pub struct GetWeightResponse {
maybe_thumbnail_token: Option<String>,
description_markdown: String,
description_rendered_html: String,
creator: UserDetailsLight,
creator_set_visibility: Visibility,
file_size_bytes: i32,
file_checksum_sha2: String,
/// If an avatar is set, this is the path to the asset.
maybe_avatar_public_bucket_path: Option<String>,
cached_user_ratings_negative_count: u32,
cached_user_ratings_positive_count: u32,
cached_user_ratings_total_count: u32,
@@ -44,6 +48,7 @@ pub struct GetWeightResponse {
version: i32,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
}
#[derive(Deserialize,ToSchema)]
@@ -137,8 +142,25 @@ pub async fn get_weight_handler(
if is_private && creator_user_token.as_str() != &user_session.user_token {
warn!("user is not allowed to view this weight: {}", user_session.user_token);
return Err(GetWeightError::NotAuthorized);
}
}
let maybe_avatar = weight.maybe_avatar_public_bucket_hash
.as_deref()
.map(|hash| {
MediaFileBucketPath::from_object_hash(
hash,
weight.maybe_public_bucket_prefix.as_deref(),
weight.maybe_public_bucket_extension.as_deref())
.get_full_object_path_str()
.to_string()
});
let creator = UserDetailsLight::from_db_fields(
&weight.creator_user_token,
&weight.creator_username,
&weight.creator_display_name,
&weight.creator_gravatar_hash,
);
let response = GetWeightResponse {
success: true,
@@ -149,6 +171,8 @@ pub async fn get_weight_handler(
maybe_thumbnail_token: weight.maybe_thumbnail_token,
description_markdown: weight.description_markdown,
description_rendered_html: weight.description_rendered_html,
maybe_avatar_public_bucket_path: maybe_avatar,
creator,
creator_set_visibility: weight.creator_set_visibility,
file_size_bytes: weight.file_size_bytes,
file_checksum_sha2: weight.file_checksum_sha2,
@@ -158,7 +182,8 @@ pub async fn get_weight_handler(
maybe_cached_user_ratings_ratio: weight.maybe_cached_user_ratings_ratio,
cached_user_ratings_last_updated_at: weight.cached_user_ratings_last_updated_at,
version: weight.version,
created_at: weight.created_at
created_at: weight.created_at,
updated_at: weight.updated_at
};
let body = serde_json::to_string(&response)
@@ -1,27 +1,23 @@
use std::sync::Arc;
use actix_web::web::Query;
use actix_web::{ HttpRequest, HttpResponse, web };
use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use chrono::{ DateTime, Utc };
use log::{ info, warn };
use chrono::{DateTime, Utc};
use log::{info, warn};
use rand::Rng;
use enums::common::visibility::Visibility;
use mysql_queries::queries::model_weights::list_weights_query_builder::ListWeightsQueryBuilder;
use enums::by_table::model_weights::{
weights_types::WeightsType,
weights_category::WeightsCategory,
};
use tokens::tokens::users::UserToken;
use tokens::tokens::model_weights::ModelWeightToken;
use crate::http_server::common_responses::user_details_lite::{UserDetailsLight,DefaultAvatarInfo};
use crate::server_state::ServerState;
use utoipa::ToSchema;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use enums::common::visibility::Visibility;
use mysql_queries::queries::model_weights::list::list_weights_query_builder::ListWeightsQueryBuilder;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::server_state::ServerState;
#[derive(Deserialize,ToSchema)]
pub struct ListAvailableWeightsQuery {
@@ -12,7 +12,7 @@ use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
};
use mysql_queries::queries::model_weights::list_weights_by_tokens::list_weights_by_tokens;
use mysql_queries::queries::model_weights::list::list_weights_by_tokens::list_weights_by_tokens;
use tokens::tokens::model_weights::ModelWeightToken;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
@@ -4,23 +4,25 @@ use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Path;
use actix_web::web::{Path, Query};
use chrono::{DateTime, Utc};
use log::warn;
use utoipa::{IntoParams, ToSchema};
use enums::common::visibility::Visibility;
use mysql_queries::queries::model_weights::list_weights_by_user::{list_weights_by_creator_username, WeightsJoinUserRecord};
use mysql_queries::queries::model_weights::list::list_weights_by_user::{list_weights_by_creator_username, ListWeightsForUserArgs};
use tokens::tokens::model_weights::ModelWeightToken;
use crate::http_server::common_responses::pagination_page::PaginationPage;
use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
use crate::server_state::ServerState;
use utoipa::ToSchema;
#[derive(Serialize, Clone,ToSchema)]
pub struct Weight {
weight_token: ModelWeightToken,
title: String,
weights_type: String,
weights_category: String,
creator: UserDetailsLight,
creator_set_visibility: Visibility,
@@ -45,9 +47,16 @@ pub struct Weight {
#[derive(Serialize,ToSchema)]
pub struct ListWeightsByUserSuccessResponse {
pub success: bool,
pub weights: Vec<Weight>,
pub results: Vec<Weight>,
pub pagination: PaginationPage,
}
#[derive(Deserialize, ToSchema, IntoParams)]
pub struct ListWeightsForUserQueryParams {
pub sort_ascending: Option<bool>,
pub page_size: Option<usize>,
pub page_index: Option<usize>,
}
#[derive(Deserialize,ToSchema)]
pub struct ListWeightsByUserPathInfo {
username: String,
@@ -84,11 +93,13 @@ impl ResponseError for ListWeightsByUserError {
),
params(
("path" = ListWeightsByUserPathInfo, description = "Payload for Request"),
ListWeightsForUserQueryParams
)
)]
pub async fn list_weights_by_user_handler(
http_request: HttpRequest,
path: Path<ListWeightsByUserPathInfo>,
query: Query<ListWeightsForUserQueryParams>,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, ListWeightsByUserError> {
@@ -112,28 +123,39 @@ pub async fn list_weights_by_user_handler(
let username = path.username.as_ref();
let creator_user_token = user_session.user_token.clone();
let is_mod = user_session.can_ban_users;
let limit = query.page_size.unwrap_or(25);
let sort_ascending = query.sort_ascending.unwrap_or(false);
let page_size = query.page_size.unwrap_or_else(|| 25);
let page_index = query.page_index.unwrap_or_else(|| 0);
let query_results = list_weights_by_creator_username(
&server_state.mysql_pool,
&username,
is_mod,
ListWeightsForUserArgs{
creator_username: username,
page_size,
page_index,
can_see_deleted: is_mod,
sort_ascending,
mysql_pool: &server_state.mysql_pool,
}
).await.map_err(|e| {
warn!("Error querying for weights: {:?}", e);
ListWeightsByUserError::ServerError
});
let weights = match query_results {
Ok(weights) => weights,
let results_page = match query_results {
Ok(results) => results,
Err(e) => {
warn!("Error querying for weights: {:?}", e);
return Err(ListWeightsByUserError::ServerError);
}
};
let weights:Vec<Weight> = weights.into_iter().map(|weight| {
let weights:Vec<Weight> = results_page.records.into_iter().map(|weight| {
Weight {
weight_token: weight.token,
title: weight.title,
weights_type: weight.weights_type.to_string(),
weights_category: weight.weights_category.to_string(),
creator: UserDetailsLight::from_db_fields(
&weight.creator_user_token,
&weight.creator_username,
@@ -159,7 +181,7 @@ pub async fn list_weights_by_user_handler(
let final_weights:Vec<Weight>;
// if it's not the user ... then only show public weights else show private and public
if (creator_user_token != user_session.user_token) {
if creator_user_token != user_session.user_token {
final_weights = weights.into_iter().filter(|weight| {
weight.creator_set_visibility == Visibility::Public
}).collect();
@@ -172,7 +194,11 @@ pub async fn list_weights_by_user_handler(
let response: ListWeightsByUserSuccessResponse = ListWeightsByUserSuccessResponse {
success: true,
weights: final_weights,
results: final_weights,
pagination: PaginationPage {
current: results_page.current_page,
total_page_count: results_page.total_page_count,
},
};
@@ -3,6 +3,7 @@ pub mod get_weight;
pub mod list_available_weights;
pub mod list_featured_weights;
pub mod list_weights_by_user;
pub mod set_model_weight_avatar;
pub mod update_weight;
// pub mod create_weight; this is a private api call internally
// pub mod drain_multipart_weights_request;
@@ -0,0 +1,179 @@
use std::fmt;
use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Path;
use log::warn;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_ip::get_request_ip;
use http_server_common::response::response_success_helpers::simple_json_success;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use mysql_queries::queries::media_files::get_media_file::get_media_file;
use mysql_queries::queries::model_weights::edit::set_model_weight_avatar::{set_model_weight_avatar, UpdateArgs};
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::model_weights::ModelWeightToken;
use crate::server_state::ServerState;
#[derive(Deserialize)]
pub struct SetModelWeightAvatarRequest {
pub avatar_media_file_token: Option<MediaFileToken>,
}
#[derive(Serialize)]
pub struct SetModelWeightAvatarResponse {
pub success: bool,
}
/// For the URL PathInfo
#[derive(Deserialize)]
pub struct SetModelWeightAvatarPathInfo {
token: ModelWeightToken,
}
// =============== Error Response ===============
#[derive(Debug, Serialize)]
pub enum SetModelWeightAvatarError {
BadInput(String),
NotFound,
NotAuthorized,
ServerError,
}
impl ResponseError for SetModelWeightAvatarError {
fn status_code(&self) -> StatusCode {
match *self {
SetModelWeightAvatarError::BadInput(_) => StatusCode::BAD_REQUEST,
SetModelWeightAvatarError::NotFound => StatusCode::NOT_FOUND,
SetModelWeightAvatarError::NotAuthorized => StatusCode::UNAUTHORIZED,
SetModelWeightAvatarError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
serialize_as_json_error(self)
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for SetModelWeightAvatarError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// =============== Handler ===============
pub async fn set_model_weight_avatar_handler(
http_request: HttpRequest,
path: Path<SetModelWeightAvatarPathInfo>,
request: web::Json<SetModelWeightAvatarRequest>,
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, SetModelWeightAvatarError>
{
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session(&http_request, &server_state.mysql_pool)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
SetModelWeightAvatarError::ServerError
})?;
let user_session = match maybe_user_session {
Some(session) => session,
None => {
warn!("not logged in");
return Err(SetModelWeightAvatarError::NotAuthorized);
}
};
let media_file_token = path.token.clone();
let is_mod = user_session.can_ban_users;
let model_weight_lookup_result = get_weight_by_token(
&path.token,
is_mod,
&server_state.mysql_pool,
).await;
let model_weight = match model_weight_lookup_result {
Ok(Some(model_weight)) => model_weight,
Ok(None) => {
warn!("Model weight not found: {:?}", &path.token);
return Err(SetModelWeightAvatarError::NotFound);
},
Err(err) => {
warn!("Error looking up model_weights : {:?}", err);
return Err(SetModelWeightAvatarError::ServerError);
}
};
let is_creator = model_weight.creator_user_token.as_str() == &user_session.user_token;
if !is_creator && !is_mod {
warn!("user is not allowed to edit this media_file: {}", user_session.user_token);
return Err(SetModelWeightAvatarError::NotAuthorized);
}
let mut maybe_set_media_file_token = None;
let delete_avatar = request.avatar_media_file_token
.as_ref()
.map(|token| token.as_str().trim().is_empty())
.unwrap_or(true);
if !delete_avatar {
if let Some(media_file_token) = &request.avatar_media_file_token {
let media_file_lookup_result = get_media_file(
&media_file_token,
false,
&server_state.mysql_pool,
).await;
let media_file = match media_file_lookup_result {
Ok(Some(model_weight)) => model_weight,
Ok(None) => {
warn!("Media file not found: {:?}", media_file_token);
return Err(SetModelWeightAvatarError::NotFound);
},
Err(err) => {
warn!("Error looking up model_weights : {:?}", err);
return Err(SetModelWeightAvatarError::ServerError);
}
};
if media_file.creator_set_visibility != Visibility::Public
|| media_file.media_type != MediaFileType::Image {
return Err(SetModelWeightAvatarError::BadInput("Invalid media file token.".to_string()));
}
maybe_set_media_file_token = Some(media_file.token);
}
}
// TODO(bt,2023-12-21): DB needs a column, or we need an ip audit log
let _ip_address = get_request_ip(&http_request);
let query_result = set_model_weight_avatar(UpdateArgs {
model_weight_token: &path.token,
maybe_avatar_media_file_token: maybe_set_media_file_token.as_ref(),
mysql_pool: &server_state.mysql_pool,
}).await;
match query_result {
Ok(_) => {},
Err(err) => {
warn!("Update MediaFile DB error: {:?}", err);
return Err(SetModelWeightAvatarError::ServerError);
}
};
Ok(simple_json_success())
}
@@ -11,7 +11,7 @@ use enums::common::visibility::Visibility;
use http_server_common::response::response_success_helpers::simple_json_success;
use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use mysql_queries::queries::model_weights::update_weight::{ update_weights, UpdateWeightArgs };
use mysql_queries::queries::model_weights::edit::update_weight::{ update_weights, UpdateWeightArgs };
use tokens::tokens::model_weights::ModelWeightToken;
use crate::server_state::ServerState;
@@ -160,6 +160,7 @@ use crate::http_server::endpoints::weights::get_weight::get_weight_handler;
use crate::http_server::endpoints::weights::list_available_weights::list_available_weights_handler;
use crate::http_server::endpoints::weights::list_featured_weights::list_featured_weights_handler;
use crate::http_server::endpoints::weights::list_weights_by_user::list_weights_by_user_handler;
use crate::http_server::endpoints::weights::set_model_weight_avatar::set_model_weight_avatar_handler;
use crate::http_server::endpoints::weights::update_weight::update_weight_handler;
pub fn add_routes<T, B> (app: App<T>, server_environment: ServerEnvironment) -> App<T>
@@ -1272,12 +1273,13 @@ fn add_weights_routes<T, B>(app: App<T>) -> App<T>
web
::scope("/v1/weights")
//.route("/upload", web::post().to(upload_weights_handler))
.service(
web
::resource("/weight/{weight_token}")
.route(web::get().to(get_weight_handler))
.route(web::post().to(update_weight_handler))
.route(web::delete().to(delete_weight_handler))
.service(web::resource("/weight/{weight_token}")
.route(web::get().to(get_weight_handler))
.route(web::post().to(update_weight_handler))
.route(web::delete().to(delete_weight_handler))
)
.service(web::resource("/weight/{token}/avatar")
.route(web::post().to(set_model_weight_avatar_handler))
)
.route("/by_user/{username}", web::get().to(list_weights_by_user_handler))
.route("/list", web::get().to(list_available_weights_handler))
+1
View File
@@ -4,6 +4,7 @@ set -euxo pipefail
echo 'Run cargo check...'
#RUSTFLAGS=-Awarnings cargo check
cargo check
echo 'Run lints...'