Seed Model weights fix

re-home nessu's models (per vegito)
fix build errors; kebab-case all CLI apps
2026-11-10 07:56:46 -05:00 · 2023-12-18 21:08:43 -05:00 · 2023-12-16 23:26:16 -05:00 · 2023-12-15 19:39:06 -05:00 · 2023-12-15 19:28:37 -05:00 · 2023-12-15 18:36:31 -05:00
116 changed files with 2221 additions and 500 deletions
@@ -3,6 +3,7 @@
 .env
 .env-aichatbot-secrets
 .env-elasticsearch-secrets
+.env-migration-tool-secrets
 .env-secrets
 _develop/tools/.secrets
 _develop/tools/db_secrets.toml
@@ -17,6 +18,9 @@ secrets.toml
 secrets.yaml
 twitch_secrets.toml

+# Old env secrets
+.*-secrets
+
 # New env secrets (per-app)
 *-secrets.env

@@ -1,6 +1,6 @@
 {
  "db_name": "MySQL",
-  "query": "\nSELECT\n    f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n    f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n    f.entity_token,\n    f.user_token as `user_token: tokens::tokens::users::UserToken`,\n    u.username,\n    u.display_name as user_display_name,\n    u.email_gravatar_hash as user_gravatar_hash,\n    f.created_at,\n    f.updated_at,\n    f.deleted_at,\n\n    media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n    media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n    tts_models.title as maybe_descriptive_text_tts_model_title,\n    tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n    users.display_name as maybe_descriptive_text_user_display_name,\n    voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n    zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n    user_bookmarks AS f\nJOIN users AS u\n    ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n    u.username = ?\n    AND f.entity_type = ?\n    AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n        ",
+  "query": "\nSELECT\n    f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n    f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n    f.entity_token,\n    f.user_token as `user_token: tokens::tokens::users::UserToken`,\n    u.username,\n    u.display_name as user_display_name,\n    u.email_gravatar_hash as user_gravatar_hash,\n    f.created_at,\n    f.updated_at,\n    f.deleted_at,\n\n    media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n    media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n    model_weights.title as maybe_descriptive_text_model_weight_title,\n    tts_models.title as maybe_descriptive_text_tts_model_title,\n    tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n    users.display_name as maybe_descriptive_text_user_display_name,\n    voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n    zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n    user_bookmarks AS f\nJOIN users AS u\n    ON f.user_token = u.token\n\nLEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n    u.username = ?\n    AND f.entity_type = ?\n    AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n        ",
  "describe": {
    "columns": [
      {
@@ -125,7 +125,7 @@
      },
      {
        "ordinal": 12,
-        "name": "maybe_descriptive_text_tts_model_title",
+        "name": "maybe_descriptive_text_model_weight_title",
        "type_info": {
          "type": "VarString",
          "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -135,6 +135,16 @@
      },
      {
        "ordinal": 13,
+        "name": "maybe_descriptive_text_tts_model_title",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 14,
        "name": "maybe_descriptive_text_tts_result_inference_text",
        "type_info": {
          "type": "Blob",
@@ -144,7 +154,7 @@
        }
      },
      {
-        "ordinal": 14,
+        "ordinal": 15,
        "name": "maybe_descriptive_text_user_display_name",
        "type_info": {
          "type": "VarString",
@@ -154,7 +164,7 @@
        }
      },
      {
-        "ordinal": 15,
+        "ordinal": 16,
        "name": "maybe_descriptive_text_voice_conversion_model_title",
        "type_info": {
          "type": "VarString",
@@ -164,7 +174,7 @@
        }
      },
      {
-        "ordinal": 16,
+        "ordinal": 17,
        "name": "maybe_descriptive_text_zs_voice_title",
        "type_info": {
          "type": "VarString",
@@ -194,8 +204,9 @@
      true,
      true,
      true,
+      true,
      true
    ]
  },
-  "hash": "d9c00b3c78ed6ec31b281c7ea5fb1efd0cfa6b20899151f9e7f1a122c1f7a70e"
+  "hash": "3a7ac7f979ac270dad1070031d0d663d0eb3d969365f6b2fd048ca41c5c455b2"
 }
@@ -1,6 +1,6 @@
 {
  "db_name": "MySQL",
-  "query": "\nSELECT\n    f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n    f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n    f.entity_token,\n    f.user_token as `user_token: tokens::tokens::users::UserToken`,\n    u.username,\n    u.display_name as user_display_name,\n    u.email_gravatar_hash as user_gravatar_hash,\n    f.created_at,\n    f.updated_at,\n    f.deleted_at,\n\n    media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n    media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n    tts_models.title as maybe_descriptive_text_tts_model_title,\n    tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n    users.display_name as maybe_descriptive_text_user_display_name,\n    voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n    zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n    user_bookmarks AS f\nJOIN users AS u\n    ON f.user_token = u.token\n\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n    u.username = ?\n    AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n        ",
+  "query": "\nSELECT\n    f.token as `token: tokens::tokens::user_bookmarks::UserBookmarkToken`,\n    f.entity_type as `entity_type: enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType`,\n    f.entity_token,\n    f.user_token as `user_token: tokens::tokens::users::UserToken`,\n    u.username,\n    u.display_name as user_display_name,\n    u.email_gravatar_hash as user_gravatar_hash,\n    f.created_at,\n    f.updated_at,\n    f.deleted_at,\n\n    media_files.media_type as `maybe_media_file_type: enums::by_table::media_files::media_file_type::MediaFileType`,\n    media_files.origin_product_category as `maybe_media_file_origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,\n\n    model_weights.title as maybe_descriptive_text_model_weight_title,\n    tts_models.title as maybe_descriptive_text_tts_model_title,\n    tts_results.raw_inference_text as maybe_descriptive_text_tts_result_inference_text,\n    users.display_name as maybe_descriptive_text_user_display_name,\n    voice_conversion_models.title as maybe_descriptive_text_voice_conversion_model_title,\n    zs_voices.title as maybe_descriptive_text_zs_voice_title\n\nFROM\n    user_bookmarks AS f\nJOIN users AS u\n    ON f.user_token = u.token\n\nLEFT OUTER JOIN model_weights ON model_weights.token = f.entity_token\nLEFT OUTER JOIN media_files ON media_files.token = f.entity_token\nLEFT OUTER JOIN tts_models ON tts_models.token = f.entity_token\nLEFT OUTER JOIN tts_results ON tts_results.token = f.entity_token\nLEFT OUTER JOIN users ON users.token = f.entity_token\nLEFT OUTER JOIN voice_conversion_models ON voice_conversion_models.token = f.entity_token\nLEFT OUTER JOIN zs_voices ON zs_voices.token = f.entity_token\n\nWHERE\n    u.username = ?\n    AND f.deleted_at IS NULL\nORDER BY f.id DESC\nLIMIT 50\n        ",
  "describe": {
    "columns": [
      {
@@ -125,7 +125,7 @@
      },
      {
        "ordinal": 12,
-        "name": "maybe_descriptive_text_tts_model_title",
+        "name": "maybe_descriptive_text_model_weight_title",
        "type_info": {
          "type": "VarString",
          "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -135,6 +135,16 @@
      },
      {
        "ordinal": 13,
+        "name": "maybe_descriptive_text_tts_model_title",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 14,
        "name": "maybe_descriptive_text_tts_result_inference_text",
        "type_info": {
          "type": "Blob",
@@ -144,7 +154,7 @@
        }
      },
      {
-        "ordinal": 14,
+        "ordinal": 15,
        "name": "maybe_descriptive_text_user_display_name",
        "type_info": {
          "type": "VarString",
@@ -154,7 +164,7 @@
        }
      },
      {
-        "ordinal": 15,
+        "ordinal": 16,
        "name": "maybe_descriptive_text_voice_conversion_model_title",
        "type_info": {
          "type": "VarString",
@@ -164,7 +174,7 @@
        }
      },
      {
-        "ordinal": 16,
+        "ordinal": 17,
        "name": "maybe_descriptive_text_zs_voice_title",
        "type_info": {
          "type": "VarString",
@@ -194,8 +204,9 @@
      true,
      true,
      true,
+      true,
      true
    ]
  },
-  "hash": "eea9531d5261833b93baadee8b99449362c074fc2397e23ecd175b6a0425b16b"
+  "hash": "707af35ee4ab4519f28e6d69982944666e6c7c0f24b5da79dde828ffdb2c8302"
 }
@@ -483,15 +483,6 @@ dependencies = [
 "libc",
 ]

-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "anstream"
 version = "0.6.4"
@@ -605,9 +596,9 @@ dependencies = [

 [[package]]
 name = "async-stripe"
-version = "0.25.2"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1b39347f478f57a40cbaf82cbaebb4a350acc757cda30fc7ff88707a11d1d81"
+checksum = "19a0b12cb3b2df9abfaa381632e4d55b2aede9063034daaa0d0145413719a7e1"
 dependencies = [
 "chrono",
 "futures-util",
@@ -684,17 +675,6 @@ dependencies = [
 "url",
 ]

-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -813,6 +793,7 @@ dependencies = [
 "tokio",
 "toml 0.5.11",
 "url_config",
+ "user_traits_component",
 ]

 [[package]]
@@ -1098,21 +1079,6 @@ dependencies = [
 "inout",
 ]

-[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "ansi_term",
- "atty",
- "bitflags 1.3.2",
- "strsim 0.8.0",
- "textwrap",
- "unicode-width",
- "vec_map",
-]
-
 [[package]]
 name = "clap"
 version = "4.4.8"
@@ -1132,7 +1098,7 @@ dependencies = [
 "anstream",
 "anstyle",
 "clap_lex",
- "strsim 0.10.0",
+ "strsim",
 ]

 [[package]]
@@ -1529,7 +1495,7 @@ dependencies = [
 "ident_case",
 "proc-macro2",
 "quote",
- "strsim 0.10.0",
+ "strsim",
 "syn 1.0.109",
 ]

@@ -1543,7 +1509,7 @@ dependencies = [
 "ident_case",
 "proc-macro2",
 "quote",
- "strsim 0.10.0",
+ "strsim",
 "syn 1.0.109",
 ]

@@ -1557,7 +1523,7 @@ dependencies = [
 "ident_case",
 "proc-macro2",
 "quote",
- "strsim 0.10.0",
+ "strsim",
 "syn 2.0.39",
 ]

@@ -1737,7 +1703,7 @@ name = "dev-database-seed"
 version = "0.0.1"
 dependencies = [
 "buckets",
- "clap 2.34.0",
+ "clap",
 "cloud_storage",
 "config",
 "easyenv",
@@ -1763,7 +1729,7 @@ name = "dev-upload-media-file"
 version = "0.0.1"
 dependencies = [
 "buckets",
- "clap 2.34.0",
+ "clap",
 "cloud_storage",
 "config",
 "easyenv",
@@ -2046,7 +2012,7 @@ name = "elasticsearch-cli"
 version = "0.0.1"
 dependencies = [
 "buckets",
- "clap 2.34.0",
+ "clap",
 "config",
 "easyenv",
 "elasticsearch",
@@ -2102,7 +2068,6 @@ dependencies = [
 "bootstrap",
 "buckets",
 "chrono",
- "clap 2.34.0",
 "cloud_storage",
 "concurrency",
 "config",
@@ -2666,15 +2631,6 @@ dependencies = [
 "unicode-segmentation",
 ]

-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.3.3"
@@ -3077,7 +3033,7 @@ version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
- "hermit-abi 0.3.3",
+ "hermit-abi",
 "rustix",
 "windows-sys 0.48.0",
 ]
@@ -3380,7 +3336,7 @@ dependencies = [
 name = "lru-delete-models"
 version = "0.0.1"
 dependencies = [
- "clap 4.4.8",
+ "clap",
 "errors",
 "logging",
 "my-workspace-hack",
@@ -3524,6 +3480,29 @@ dependencies = [
 "my-workspace-hack",
 ]

+[[package]]
+name = "migration-tool"
+version = "0.0.1"
+dependencies = [
+ "buckets",
+ "clap",
+ "config",
+ "easyenv",
+ "enums",
+ "errors",
+ "log",
+ "my-workspace-hack",
+ "mysql_queries",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "sqlx",
+ "storyteller_root",
+ "strum",
+ "tokens",
+ "tokio",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -3973,7 +3952,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.3.3",
+ "hermit-abi",
 "libc",
 ]

@@ -5125,7 +5104,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 name = "scrape_test"
 version = "0.0.1"
 dependencies = [
- "clap 2.34.0",
+ "clap",
 "errors",
 "my-workspace-hack",
 "tokio",
@@ -5954,6 +5933,7 @@ dependencies = [
 "twitch_oauth2",
 "url_config",
 "user_input_common",
+ "user_traits_component",
 "users_component",
 "utoipa",
 "utoipa-swagger-ui",
@@ -6006,12 +5986,6 @@ dependencies = [
 "unicode-normalization",
 ]

-[[package]]
-name = "strsim"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -6331,15 +6305,6 @@ dependencies = [
 "my-workspace-hack",
 ]

-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.50"
@@ -6764,7 +6729,6 @@ version = "0.0.1"
 dependencies = [
 "anyhow",
 "chrono",
- "clap 2.34.0",
 "cloud_storage",
 "collections",
 "config",
@@ -7066,6 +7030,16 @@ dependencies = [
 "unicode-segmentation",
 ]

+[[package]]
+name = "user_traits_component"
+version = "0.0.1"
+dependencies = [
+ "actix-web",
+ "errors",
+ "mockall",
+ "my-workspace-hack",
+]
+
 [[package]]
 name = "users_component"
 version = "0.0.1"
@@ -7109,6 +7083,7 @@ dependencies = [
 "tokens",
 "toml 0.5.11",
 "user_input_common",
+ "user_traits_component",
 ]

 [[package]]
@@ -7185,12 +7160,6 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"

-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
 [[package]]
 name = "version-sync"
 version = "0.9.5"
@@ -10,13 +10,14 @@ members = [
  "crates/_types/tokens",

  # CLI tools for operations, etc.
-  "crates/cli/database_backup",
-  "crates/cli/database_cleanup",
-  "crates/cli/dev_database_seed",
-  "crates/cli/dev_upload_media_file",
+  "crates/cli/database-backup",
+  "crates/cli/database-cleanup",
+  "crates/cli/dev-database-seed",
+  "crates/cli/dev-upload-media-file",
  "crates/cli/elasticsearch-cli",
  "crates/cli/kubectl-pod-cleanup",
  "crates/cli/lru-delete-models",
+  "crates/cli/migration-tool",
  "crates/cli/scrape_test",

  # Shared libraries
@@ -83,6 +84,7 @@ members = [
  # Service Plugins: complete subsystems that can be plugged into services (users, billing, etc.)
  "crates/service/plugins/billing",
  "crates/service/plugins/inference_workloads",
+  "crates/service/plugins/user_traits_component",
  "crates/service/plugins/users",

  # Vendored code
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Re-home Nessu's models
+update tts_models set creator_user_token = "user_7z1h6sykg4r62" where creator_user_token = "U:C00NRSW1AE8T4" limit 5;
@@ -13,6 +13,9 @@ where token IN (
    'vcm_502wkt2pca3y'
 );

+-- Re-home Nessu's models
+update voice_conversion_models set creator_user_token = "user_7z1h6sykg4r62" where creator_user_token = "U:C00NRSW1AE8T4" limit 5;
+
 -- Remove broken models
 update voice_conversion_models
 set
@@ -376,6 +379,15 @@ where token IN (
 );


+update voice_conversion_models
+set
+  mod_deleted_at = NOW(),
+  maybe_mod_comments = 'abuse'
+where token IN (
+  'vcm_qajyd528p8kw',
+  'vcm_f5pcdecmzcc5'
+);
+



@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights DROP COLUMN maybe_migration_old_model_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    ADD COLUMN maybe_migration_old_model_token VARCHAR(32) DEFAULT NULL
+    AFTER cached_user_ratings_last_updated_at;
+
+ALTER TABLE model_weights
+    ADD UNIQUE INDEX unique_index_maybe_migration_old_model_token (maybe_migration_old_model_token);
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE voice_conversion_models
+    DROP COLUMN maybe_migration_new_model_weights_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE voice_conversion_models
+    ADD COLUMN maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL
+    AFTER maybe_mod_comments;
+
+ALTER TABLE voice_conversion_models
+    ADD UNIQUE INDEX unique_index_maybe_migration_new_model_weights_token (maybe_migration_new_model_weights_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE tts_models DROP COLUMN maybe_migration_new_model_weights_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE tts_models
+    ADD COLUMN maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL
+    AFTER maybe_mod_user_token;
+
+ALTER TABLE tts_models
+    ADD UNIQUE INDEX unique_index_maybe_migration_new_model_weights_token (maybe_migration_new_model_weights_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE vocoder_models DROP COLUMN maybe_migration_new_model_weights_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE vocoder_models
+    ADD COLUMN maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL
+    AFTER maybe_mod_user_token;
+
+ALTER TABLE vocoder_models
+    ADD UNIQUE INDEX unique_index_maybe_migration_new_model_weights_token (maybe_migration_new_model_weights_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights DROP COLUMN maybe_avatar_media_file_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    ADD COLUMN maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL
+    AFTER maybe_thumbnail_token;
+
+ALTER TABLE model_weights
+    ADD INDEX index_maybe_avatar_media_file_token (maybe_avatar_media_file_token);
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights DROP COLUMN maybe_cover_media_file_token;
+
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    ADD COLUMN maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL
+    AFTER maybe_avatar_media_file_token;
+
+ALTER TABLE model_weights
+    ADD INDEX index_maybe_cover_media_file_token (maybe_cover_media_file_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users DROP COLUMN maybe_avatar_media_file_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users
+    ADD COLUMN maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL
+    AFTER ip_address_last_update;
+
+ALTER TABLE users
+    ADD INDEX index_maybe_avatar_media_file_token (maybe_avatar_media_file_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users DROP COLUMN maybe_cover_media_file_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users
+    ADD COLUMN maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL
+    AFTER maybe_avatar_media_file_token;
+
+ALTER TABLE users
+    ADD INDEX index_maybe_cover_media_file_token (maybe_cover_media_file_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP TABLE model_weights_preview_samples;
@@ -0,0 +1,47 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE model_weights_preview_samples (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- ========== WEIGHTS CLASSIFICATION ==========
+
+  -- Foreign key to model weights.
+  -- This is in a composite index with media_file_token to make sure a media file only applies
+  -- to a given model once.
+  model_weight_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to media files
+  -- This is in a composite index with model_weight_token to make sure a media file only applies
+  -- to a given model once.
+  media_file_token VARCHAR(32) NOT NULL,
+
+  -- Information about who set the sample
+  --  * automation
+  --  * author
+  --  * community
+  set_by VARCHAR(16) NOT NULL,
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (model_weight_token, media_file_token),
+
+  KEY index_model_weight_token (model_weight_token),
+  KEY index_media_file_token (media_file_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,9 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- I'm doing this to visually align these tables and sort them together, because there will likely be other model_weights_* tables.
+RENAME TABLE model_weights_extension_image_generation_details TO model_weights_image_generation_details;
+RENAME TABLE model_weights_extension_tts_details TO model_weights_tts_details;
+RENAME TABLE model_weights_extension_vocoder_details TO model_weights_vocoder_details;
+RENAME TABLE model_weights_extension_voice_conversion_details TO model_weights_voice_conversion_details;
@@ -0,0 +1,9 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- I'm doing this to visually align these tables and sort them together, because there will likely be other model_weights_* tables.
+RENAME TABLE model_weights_image_generation_details TO model_weights_extension_image_generation_details;
+RENAME TABLE model_weights_tts_details TO model_weights_extension_tts_details;
+RENAME TABLE model_weights_vocoder_details TO model_weights_extension_vocoder_details;
+RENAME TABLE model_weights_voice_conversion_details TO model_weights_extension_voice_conversion_details;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP INDEX index_weights_type ON model_weights;
+DROP INDEX index_weights_category ON model_weights;
+DROP INDEX fk_creator_user_token ON model_weights;
+DROP INDEX index_creator_set_visibility ON model_weights;
+DROP INDEX fk_maybe_avatar_media_file_token ON model_weights;
+DROP INDEX fk_maybe_cover_media_file_token ON model_weights;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE INDEX index_weights_type ON model_weights (weights_type);
+CREATE INDEX index_weights_category ON model_weights (weights_category);
+CREATE INDEX fk_creator_user_token ON model_weights (creator_user_token);
+CREATE INDEX index_creator_set_visibility ON model_weights (creator_set_visibility);
+CREATE INDEX fk_maybe_avatar_media_file_token ON model_weights (maybe_avatar_media_file_token);
+CREATE INDEX fk_maybe_cover_media_file_token ON model_weights (maybe_cover_media_file_token);
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP INDEX fk_maybe_avatar_media_file_token ON users;
+DROP INDEX fk_maybe_cover_media_file_token ON users;
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE INDEX fk_maybe_avatar_media_file_token ON users (maybe_avatar_media_file_token);
+CREATE INDEX fk_maybe_cover_media_file_token ON users (maybe_cover_media_file_token);
@@ -0,0 +1,200 @@
+-- NB: This is a manually squashed view of all the CREATE and ALTER statements,
+-- with comments attached to the fields for centralized documentation.
+
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Every type of fine tuned model that users can upload.
+CREATE TABLE model_weights (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- ========== WEIGHTS CLASSIFICATION ==========
+
+  -- The exact type of weights
+  -- Currently supported values:
+  --   * 'hifigan_tt2' - hifigan for Tacotron2; TTS vocoder
+  --   * 'rvc_v2' - RVC (v2); voice conversion
+  --   * 'sd_1.5' - Stable Diffusion 1.5; image generation
+  --   * 'sdxl' - Stable Diffusion XL; image generation
+  --   * 'so_vits_svc' - SVC; voice conversion
+  --   * 'tt2' - Tacotron2; TTS
+  weights_type VARCHAR(32) NOT NULL,
+
+  -- The broader category of the weights
+  -- Currently supported values:
+  --   * 'image_generation' (SD1.5, SDXL)
+  --   * 'text_to_speech' (TT2)
+  --   * 'vocoder' (Hifigan)
+  --   * 'voice_conversion' (RVCv2, SVC)
+  weights_category VARCHAR(32) NOT NULL,
+
+
+  -- ========== MODEL DESCRIPTION ==========
+
+  -- The "name" of the model, which might be complicated.
+  title VARCHAR(255) NOT NULL,
+
+  -- TODO(bt, 2023-12-14): Delete the `maybe_thumbnail_token` column later.
+  --   We're using other columns instead.
+  maybe_thumbnail_token VARCHAR(32) DEFAULT NULL,
+
+  -- The "avatar" image is a media file of type image that serves as a
+  -- small avatar or profile picture icon.
+  maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL,
+
+  -- The "cover" image is a media file of type image that covers the
+  -- top of the page.
+  maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL,
+
+  -- The description of the model in markdown.
+  description_markdown TEXT NOT NULL,
+
+  -- Generated HTML (not user-editable).
+  description_rendered_html TEXT NOT NULL,
+
+
+  -- ========== CREATOR DETAILS ==========
+
+  -- The person that created the model.
+  creator_user_token VARCHAR(32) NOT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6.
+  -- IP address of creator at time of creation.
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- The synthetic id associated with this model.
+  -- These ids are incremented on a per-user basis to help users
+  -- sequence their own work. They serve no other purpose.
+  -- "weights" synthetic ids are incremented over all media files
+  -- creator_weights_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- "weights_type" synthetic ids are incremented over a given weight type
+  -- creator_weights_type_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- The last user to edit the model. This could be the creator or a moderator.
+  -- Check the future audit logs table for more info.
+  maybe_last_update_user_token VARCHAR(32) DEFAULT NULL,
+
+
+  -- ========== IMMUTABLE PROVENANCE AND METADATA ==========
+
+  -- Where the file was originally downloaded (if it was downloaded)
+  original_download_url VARCHAR(512) DEFAULT NULL,
+
+  -- The filename that was used at upload time (if available)
+  original_filename VARCHAR(255) DEFAULT NULL,
+
+  -- File characteristics
+  file_size_bytes INT(10) NOT NULL DEFAULT 0,
+
+  -- Checksum of the original weights file
+  -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
+  -- Note that SHA2 is a hash family (SHA-228, SHA-256, SHA-384, SHA-512, SHA-512/224, SHA-512/256, ...)
+  -- and produces different lengths output depending on the choice of algorithm.
+  file_checksum_sha2 CHAR(64) NOT NULL,
+
+  -- ========== BUCKET STORAGE ==========
+
+  -- The hash for the bucket directory that contains the original upload
+  -- as well as any associated other files.
+  -- The bucket filename for the primary file (not including the directory path) is given by:
+  -- `[{maybe_private_bucket_prefix}]{private_bucket_hash}[{maybe_private_bucket_extension}]`
+  private_bucket_hash  VARCHAR(32) NOT NULL,
+
+  -- An optional prefix on the bucket filename.
+  -- If present, this will be prepended to the beginning of the bucket filename to access the file.
+  -- The bucket filename for the primary file (not including the directory path) is given by:
+  -- `[{maybe_private_bucket_prefix}]{private_directory_hash}[{maybe_private_bucket_extension}]`
+  maybe_private_bucket_prefix VARCHAR(16) DEFAULT NULL,
+
+  -- An optional appended extension on the bucket filename.
+  -- If present, this will be appended to the end of the bucket filename to access the file.
+  -- To allow for flexibility, this extension typically will contain a leading period if
+  -- the file needs it (eg ".mp4" rather than "mp4")!
+  -- The bucket filename for the primary file (not including the directory path) is given by:
+  -- `[{maybe_private_bucket_prefix}]{private_directory_hash}[{maybe_private_bucket_extension}]`
+  maybe_private_bucket_extension VARCHAR(16) DEFAULT NULL,
+
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- Mods may have to approve of vc models for them to show up in a public index.
+  -- Models can be used by the author (or other parties that know the URL) before
+  -- they're approved, but unapproved models won't show up in public indices.
+  -- This is independent of other visibility controls (is_mod_disabled, deleted_at,
+  -- and creator_set_visibility)
+  -- is_public_listing_approved BOOLEAN DEFAULT NULL,
+
+  -- Extremely popular models may be locked from deletion or modification by users.
+  -- is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE,
+
+
+  -- ========== RATING AND RANKING ==========
+
+  -- User ratings are cached on the model, but are *not* a source of truth.
+  -- Total count only includes "positive" and "negative" votes, not neutral ones.
+  cached_user_ratings_total_count INT(10) UNSIGNED NOT NULL DEFAULT 0,
+
+  -- Total positive
+  cached_user_ratings_positive_count INT(10) UNSIGNED NOT NULL DEFAULT 0,
+
+  -- Total negative
+  cached_user_ratings_negative_count INT(10) UNSIGNED NOT NULL DEFAULT 0,
+
+  -- User ratings are cached on the model, but are *not* a source of truth.
+  -- Total count only includes "positive" and "negative" votes, not neutral ones.
+  maybe_cached_user_ratings_ratio FLOAT,
+
+  -- When the cached ratings were last calculated and updated.
+  cached_user_ratings_last_updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+
+  -- ========== MIGRATION DETAILS ==========
+
+  -- If migrated from another table, this is the token of the original model.
+  maybe_migration_old_model_token VARCHAR(32) DEFAULT NULL,
+
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+
+  KEY index_weights_type (weights_type),
+  KEY index_weights_category (weights_category),
+  KEY fk_creator_user_token (creator_user_token),
+  KEY index_creator_set_visibility (creator_set_visibility),
+  KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token),
+  KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,50 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Every type of fine tuned model that users can upload.
+CREATE TABLE model_weights_extension_image_generation_details (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Simultaneous primary and foreign key.
+  -- This should match a single record in the `model_weights` table.
+  -- This turns this table into a polymorphic extension table.
+  model_weights_token VARCHAR(32) NOT NULL,
+
+
+  -- ========== IMAGE GENERATION MODEL METADATA ==========
+
+  -- RVC (v2) specific - whether the model has an index file associated with it
+  -- These files improve the quality of the results.
+  vc_has_index_file BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (model_weights_token)
+  -- TODO:
+  -- KEY index_model_type (model_type),
+  -- KEY fk_maybe_vocoder_token (maybe_vocoder_token),
+  -- KEY fk_creator_user_token (creator_user_token),
+  -- KEY index_creator_set_visibility (creator_set_visibility),
+  -- KEY index_private_bucket_hash (private_bucket_hash),
+  -- KEY index_is_public_listing_approved (is_public_listing_approved),
+  -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,73 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Extra metadata on tts models
+CREATE TABLE model_weights_extension_tts_details (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Simultaneous primary and foreign key.
+  -- This should match a single record in the `model_weights` table.
+  -- This turns this table into a polymorphic extension table.
+  model_weights_token VARCHAR(32) NOT NULL,
+
+  -- ========== TTS MODEL METADATA ==========
+
+  -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
+  -- (Not that it matters apart from categorization, since voice conversion is universal.)
+  ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
+
+  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
+  -- (Not that it matters apart from categorization, since voice conversion is universal.)
+  ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
+
+  -- This is for Tacotron2, specifically.
+  -- The name of the type of pipeline to use, eg "legacy_fakeyou", "legacy_vocodes", "english_v1", etc.
+  -- New pipelines may be added at any point in the future and the source of truth for them will mostly
+  -- live in the `storyteller-ml` repository.
+  -- Older models and brand new models will have this field set to null, and we'll infer which value to
+  -- lazily backfill based on the creation date and language setting.
+  -- tts_text_pipeline_type VARCHAR(64) DEFAULT NULL,
+
+  -- If true, multiply the mel outputs before being vocoded by a globally default constant.
+  use_default_mel_multiply_factor BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- If not null, multiply the mel outputs before being vocoded by this value.
+  -- This is used instead of the default if `tts_use_default_mel_multiply_factor` is
+  -- set (ie. `tts_use_default_mel_multiply_factor` is ignored and this custom value is used instead)
+  maybe_custom_mel_multiply_factor DOUBLE DEFAULT NULL,
+
+  -- If set, use a different pretrained vocoder.
+  -- If not set, use the website default. (Currently HifiGan-SS)
+  maybe_default_pretrained_vocoder VARCHAR(64) DEFAULT NULL,
+
+  -- If set, we'll use a custom vocoder (from table vocoder_models) instead of a default vocoder.
+  -- If not set, we'll use `maybe_default_pretrained_vocoder` (or the website default).
+  maybe_custom_vocoder_token VARCHAR(32) DEFAULT NULL,
+
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (model_weights_token)
+  -- TODO:
+  -- KEY index_model_type (model_type),
+  -- KEY fk_maybe_vocoder_token (maybe_vocoder_token),
+  -- KEY fk_creator_user_token (creator_user_token),
+  -- KEY index_creator_set_visibility (creator_set_visibility),
+  -- KEY index_private_bucket_hash (private_bucket_hash),
+  -- KEY index_is_public_listing_approved (is_public_listing_approved),
+  -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,50 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Every type of fine tuned model that users can upload.
+CREATE TABLE model_weights_extension_vocoder_details (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Simultaneous primary and foreign key.
+  -- This should match a single record in the `model_weights` table.
+  -- This turns this table into a polymorphic extension table.
+  model_weights_token VARCHAR(32) NOT NULL,
+
+
+  -- ========== VOCODER MODEL METADATA ==========
+
+  -- Whether the vocoder is recommended for use
+  -- vocoder_is_staff_recommended BOOLEAN NOT NULL DEFAULT FALSE,
+
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (model_weights_token)
+  -- TODO:
+  -- KEY index_model_type (model_type),
+  -- KEY fk_maybe_vocoder_token (maybe_vocoder_token),
+  -- KEY fk_creator_user_token (creator_user_token),
+  -- KEY index_creator_set_visibility (creator_set_visibility),
+  -- KEY index_private_bucket_hash (private_bucket_hash),
+  -- KEY index_is_public_listing_approved (is_public_listing_approved),
+  -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,62 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Every type of fine tuned model that users can upload.
+CREATE TABLE model_weights_extension_voice_conversion_details (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Simultaneous primary and foreign key.
+  -- This should match a single record in the `model_weights` table.
+  -- This turns this table into a polymorphic extension table.
+  model_weights_token VARCHAR(32) NOT NULL,
+
+
+  -- ========== VOICE CONVERSION MODEL METADATA ==========
+
+  -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
+  -- (Not that it matters apart from categorization, since voice conversion is universal.)
+  ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
+
+  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
+  -- (Not that it matters apart from categorization, since voice conversion is universal.)
+  ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
+
+  -- RVC (v2) specific - whether the model has an index file associated with it
+  -- These files improve the quality of the results.
+  has_index_file BOOLEAN NOT NULL DEFAULT FALSE,
+
+  -- Pitch algorithm to use by default
+  default_pitch_algorithm VARCHAR(64) DEFAULT NULL,
+
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (model_weights_token)
+  -- TODO:
+  -- KEY index_model_type (model_type),
+  -- KEY fk_maybe_vocoder_token (maybe_vocoder_token),
+  -- KEY fk_creator_user_token (creator_user_token),
+  -- KEY index_creator_set_visibility (creator_set_visibility),
+  -- KEY index_private_bucket_hash (private_bucket_hash),
+  -- KEY index_is_public_listing_approved (is_public_listing_approved),
+  -- KEY index_is_locked_from_user_modification (is_locked_from_user_modification)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -0,0 +1,47 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE TABLE model_weights_preview_samples (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- ========== WEIGHTS CLASSIFICATION ==========
+
+  -- Foreign key to model weights.
+  -- This is in a composite index with media_file_token to make sure a media file only applies
+  -- to a given model once.
+  model_weight_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to media files
+  -- This is in a composite index with model_weight_token to make sure a media file only applies
+  -- to a given model once.
+  media_file_token VARCHAR(32) NOT NULL,
+
+  -- Information about who set the sample
+  --  * automation
+  --  * author
+  --  * community
+  set_by VARCHAR(16) NOT NULL,
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod.
+  -- It completely disappears from the system.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (model_weight_token, media_file_token),
+
+  KEY index_model_weight_token (model_weight_token),
+  KEY index_media_file_token (media_file_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -200,6 +200,11 @@ CREATE TABLE tts_models (
  -- The last moderator that made changes.
  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,

+  -- ========== MIGRATION DETAILS ==========
+
+  -- If migrated to model_weights, this is the new token
+  maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL,
+
  -- ========== VECTOR CLOCK ==========

  -- Incremented with every update.
@@ -99,6 +99,11 @@ CREATE TABLE vocoder_models (
  -- The last moderator that made changes.
  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,

+  -- ========== MIGRATION DETAILS ==========
+
+  -- If migrated to model_weights, this is the new token
+  maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL,
+
  -- ========== VECTOR CLOCK ==========

  -- Incremented with every update.
@@ -53,6 +53,14 @@ CREATE TABLE users (

  -- ========== DISPLAY / PROFILE ==========

+  -- The "avatar" image is a media file of type image that serves as a
+  -- small avatar or profile picture icon.
+  maybe_avatar_media_file_token VARCHAR(32) DEFAULT NULL,
+
+  -- The "cover" image is a media file of type image that covers the
+  -- top of the page.
+  maybe_cover_media_file_token VARCHAR(32) DEFAULT NULL,
+
  -- Gravatar image hashes are precomputed.
  email_gravatar_hash CHAR(32) NOT NULL,

@@ -162,7 +170,9 @@ CREATE TABLE users (
  UNIQUE KEY (token),
  UNIQUE KEY (username),
  UNIQUE KEY (email_address),
-  KEY fk_user_role_slug (user_role_slug)
+  KEY fk_user_role_slug (user_role_slug),
+  KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token),
+  KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token)

 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

@@ -121,6 +121,11 @@ CREATE TABLE voice_conversion_models (
  -- If a moderator has comments.
  maybe_mod_comments VARCHAR(255) DEFAULT NULL,

+  -- ========== MIGRATION DETAILS ==========
+
+  -- If migrated to model_weights, this is the new token
+  maybe_migration_new_model_weights_token VARCHAR(32) DEFAULT NULL,
+
  -- ========== VECTOR CLOCK ==========

  -- Incremented with every update.
@@ -0,0 +1,3 @@
+pub trait descriptor {
+    fn my_method(&self);
+}
@@ -0,0 +1,4 @@
+pub mod bucket_directory;
+pub mod bucket_file_path;
+pub mod web_file_manager;
+
@@ -0,0 +1,32 @@
+
+pub struct FileBucketPath {
+    pub bucket_clients: BucketClients
+}
+
+impl web_file_manager {
+    pub fn new(my_field: i32) -> Self {
+        web_file_manager { my_field }
+    }
+
+    pub fn increment(&mut self) {
+        self.my_field += 1;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_new() {
+        let my_class = web_file_manager::new(10);
+        assert_eq!(my_class.my_field, 10);
+    }
+
+    #[test]
+    fn test_increment() {
+        let mut my_class = web_file_manager::new(10);
+        my_class.increment();
+        assert_eq!(my_class.my_field, 11);
+    }
+}
@@ -2,4 +2,5 @@ pub mod media_files;
 pub mod media_uploads;
 pub mod public_path;
 pub mod voice_conversion_results;
-pub mod weight_uploads;
+pub mod weight_uploads;
+pub mod weight_files;
@@ -0,0 +1,75 @@
+use crockford::crockford_entropy_lower;
+
+use crate::public::public_path::PublicPath;
+use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
+
+// TODO: Generate these from a macro.
+// TODO: Use a central path registry for quick reference
+const WEIGHT_FILE_DIRECTORY: &str = "/weights";
+
+/// Directory for user weight uploads.
+/// Each uploaded file gets its own directory so that we can store the original
+/// file alongside re-processed derivatives.
+#[derive(Clone)]
+pub struct WeightFileBucketDirectory {
+  object_hash: String,
+  directory: String,
+}
+
+impl PublicPath for WeightFileBucketDirectory {}
+
+impl WeightFileBucketDirectory {
+
+  pub fn generate_new() -> Self {
+    let entropy = crockford_entropy_lower(32);
+    Self::from_object_hash(&entropy)
+  }
+
+  pub fn from_object_hash(object_hash: &str) -> Self {
+    // TODO: Path construction could be cleaner.
+    let middle = hashed_directory_path_long_string(object_hash);
+    let directory = format!("{}/{}{}", WEIGHT_FILE_DIRECTORY, middle, object_hash);
+    Self {
+      object_hash: object_hash.to_string(),
+      directory,
+    }
+  }
+
+  pub fn get_directory_path_str(&self) -> &str {
+    &self.directory
+  }
+
+  pub fn get_object_hash(&self) -> &str {
+    &self.object_hash
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+
+  #[test]
+  pub fn generate_new_entropy() {
+    let directory = WeightFileBucketDirectory::generate_new();
+    assert_eq!(directory.get_object_hash().len(), 32);
+  }
+
+  #[test]
+  pub fn get_directory_path_str() {
+    let directory = WeightFileBucketDirectory::from_object_hash("abcdefghijk");
+    assert_eq!(directory.get_directory_path_str(), "/weights/a/b/c/d/e/abcdefghijk");
+  }
+
+  #[test]
+  pub fn get_directory_path_str_short_name() {
+    let directory = WeightFileBucketDirectory::from_object_hash("foo");
+    assert_eq!(directory.get_directory_path_str(), "/weights/f/o/foo");
+  }
+
+  #[test]
+  pub fn get_object_hash() {
+    let hash = "abcdefghijk";
+    let directory = WeightFileBucketDirectory::from_object_hash(hash);
+    assert_eq!(directory.get_object_hash(), hash);
+  }
+}
@@ -0,0 +1,142 @@
+use std::path::PathBuf;
+
+use crockford::crockford_entropy_lower;
+use crate::public::public_path::PublicPath;
+
+const ORIGINAL_FILE_BASENAME : &str = "file.bin";
+
+// TODO: Generate these from a macro.
+use crate::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+/// The original user upload file.
+/// It may have derivative files (down samples, crops, etc.) that live alongside it.
+#[derive(Clone)]
+pub struct WeightFileBucketPath {
+  directory: WeightFileBucketDirectory,
+
+  filename: String,
+  full_object_path: String,
+
+  optional_prefix: Option<String>,
+  /// NB: Extension contains the leading period, eg. ".mp4"
+  optional_extension: Option<String>,
+}
+
+impl PublicPath for WeightFileBucketPath {}
+
+impl WeightFileBucketPath {
+
+  pub fn generate_new(optional_prefix: Option<&str>, optional_extension: Option<&str>) -> Self {
+    let entropy = crockford_entropy_lower(32);
+    Self::from_object_hash(&entropy, optional_prefix, optional_extension)
+  }
+
+  pub fn from_object_hash(hash: &str, optional_prefix: Option<&str>, optional_extension: Option<&str>) -> Self {
+    // TODO: Path construction could be cleaner.
+    let directory = WeightFileBucketDirectory::from_object_hash(hash);
+
+    let filename = match (optional_prefix, optional_extension) {
+      (None, None) => ORIGINAL_FILE_BASENAME.to_string(),
+      (None, Some(ext)) => format!("{}{}", hash, ext),
+      (Some(pre), None) => format!("{}{}", pre, hash),
+      (Some(pre), Some(ext)) => format!("{}{}{}", pre, hash, ext),
+    };
+
+    let full_object_path = format!("{}/{}", directory.get_directory_path_str(), filename);
+
+    Self {
+      directory,
+      filename,
+      full_object_path,
+      optional_prefix: optional_prefix.map(|p| p.to_string()),
+      optional_extension: optional_extension.map(|e| e.to_string()),
+    }
+  }
+
+  pub fn get_full_object_path_str(&self) -> &str {
+    &self.full_object_path
+  }
+
+  pub fn to_full_object_pathbuf(&self) -> PathBuf {
+    PathBuf::from(&self.full_object_path)
+  }
+
+  pub fn get_directory(&self) -> &WeightFileBucketDirectory {
+    &self.directory
+  }
+
+  pub fn get_object_hash(&self) -> &str {
+    &self.directory.get_object_hash()
+  }
+
+  pub fn get_basename(&self) -> &str {
+    &self.filename
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use std::path::PathBuf;
+
+  use crate::public::weight_files::bucket_file_path::WeightFileBucketPath;
+  mod with_prefix_and_extension {
+    use super::*;
+
+    #[test]
+    pub fn generate_new_entropy() {
+      let file = WeightFileBucketPath::generate_new(Some("pre_"), Some(".mp4"));
+      assert_eq!(file.get_object_hash().len(), 32);
+      assert_eq!(file.get_object_hash().len(), 32);
+      assert_eq!(file.get_directory().get_object_hash().len(), 32);
+    }
+
+    #[test]
+    pub fn get_full_object_path_str() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", Some("pre_"), Some(".mp4"));
+      assert_eq!(file.get_full_object_path_str(), "/weights/a/b/c/d/e/abcdefghijk/pre_abcdefghijk.mp4");
+      assert_eq!(file.get_basename(), "pre_abcdefghijk.mp4");
+    }
+  }
+
+  mod without_prefix_and_extension {
+    use super::*;
+
+    #[test]
+    pub fn generate_new_entropy() {
+      let file = WeightFileBucketPath::generate_new(None, None);
+      assert_eq!(file.get_object_hash().len(), 32);
+      assert_eq!(file.get_directory().get_object_hash().len(), 32);
+    }
+
+    #[test]
+    pub fn get_full_object_path_str() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None);
+      assert_eq!(file.get_full_object_path_str(), "/weights/a/b/c/d/e/abcdefghijk/file.bin");
+    }
+
+    #[test]
+    pub fn to_full_object_pathbuf() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None);
+      assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/weights/a/b/c/d/e/abcdefghijk/file.bin"));
+    }
+
+    #[test]
+    pub fn get_full_object_path_str_short_name() {
+      let file = WeightFileBucketPath::from_object_hash("foo", None, None);
+      assert_eq!(file.get_full_object_path_str(), "/weights/f/o/foo/file.bin");
+    }
+
+    #[test]
+    pub fn get_full_object_path_str_starts_with_directory() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None);
+      let full_path = file.get_full_object_path_str();
+      assert!(full_path.starts_with(file.get_directory().get_directory_path_str()))
+    }
+
+    #[test]
+    pub fn get_object_hash() {
+      let hash = "abcdefghijk";
+      let file = WeightFileBucketPath::from_object_hash(hash, None, None);
+      assert_eq!(file.get_object_hash(), hash);
+    }
+  }
+}
@@ -0,0 +1,2 @@
+pub mod bucket_directory;
+pub mod bucket_file_path;
@@ -27,11 +27,11 @@ impl DeterministicRng {
  }

  /// Reset the deterministic RNG's seed.
-  pub fn reset_rng(&self) {
+  pub fn reset_rng(&self, state: u64) {
    match self.rng.write() {
      Err(err) => panic!(format!("test panic due to rng failure {err}")),
      Ok(mut lock) => {
-        *lock = StdRng::seed_from_u64(0);
+        *lock = StdRng::seed_from_u64(state);
      }
    }
  }
@@ -63,9 +63,9 @@ macro_rules! impl_crockford_generator {
      /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding.
      /// This will not use a real RNG, so do not use in production code!
      /// Once called, it will output the same random numbers in sequence.
-      pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously() {
+      pub fn reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(state: u64) {
        let deterministic = crate::deterministic_rng::DeterministicRng::get_instance().expect("tests should not fail due to mutex");
-        deterministic.reset_rng();
+        deterministic.reset_rng(state);
      }

      /// Constructor for a new token to be used **DETERMINISTICALLY** in tests and seeding.
@@ -23,23 +23,18 @@ mod interface {

  #[test]
  fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_1() {
-    MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    // NB: Using the same reset seed will produce the same results each time
+    MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0);
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab");
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h");
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e");
-  }
-
-  #[test]
-  fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_2() {
-    MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    // NB: Same seed -> same tokens generated
+    MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0);
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab");
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h");
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e");
-  }
-
-  #[test]
-  fn generate_for_testing_and_dev_seeding_never_use_in_production_seriously_3() {
-    MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    // Once more...
+    MediaFileToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(0);
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_q8sz47gmfw2zx02snrbz88ns9m16ab");
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_ma1xetxrwbh39vg639a9zrq8b9wk6h");
    assert_eq!(MediaFileToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously().as_str(), "m_4tswec8z27wnm01njypx4vmfhgj41e");
@@ -33,7 +33,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }

 # External
 log = "0.4.14"
-clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+clap = "4.4.8"
 rand = "0.8.5" 
 # Runtime
 # NB: We switched to Tokio from `async-std` for broader ecosystem support.
@@ -0,0 +1,17 @@
+use clap::Parser;
+
+use errors::AnyhowResult;
+
+#[derive(Parser, Debug)]
+#[command(name="dev-database-seed")]
+pub struct CliArgs {
+  #[arg(name="seed_cloud_bucket", long="bucket", help="Seed the cloud bucket with files", required=false, default_value_t=false)]
+  pub seed_cloud_bucket: bool,
+
+  #[arg(name="seed_elasticsearch", long="elasticsearch", help="Seed the local elasticsearch", required=false, default_value_t=false)]
+  pub seed_elasticsearch: bool,
+}
+
+pub fn parse_cli_args() -> AnyhowResult<CliArgs> {
+  Ok(CliArgs::parse())
+}
@@ -132,7 +132,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                - Gawr Gura MOS 4
                "#;

-                model_weight_token = ModelWeightToken(format!("{}", i));
+                model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                title = format!("Gawr Gura: {}", i);
                description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i);
                description_rendered_html = rvc_markdown_description;
@@ -155,7 +155,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
            },
            21..=40 =>{
                // TTS
-                model_weight_token = ModelWeightToken(format!("{}", i));
+                model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                title = format!("dragonfruitGTv1: {}", i);
                
                description = format!("Description Number {}:", i);
@@ -187,7 +187,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
            },
            41..=60 => {
                // SD 1.5
-                model_weight_token = ModelWeightToken(format!("{}", i));
+                model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                title = format!("dragonfruitGTv1: {}", i);
                
                description = format!("Description Number {}:", i);
@@ -232,7 +232,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                "#;
                let sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg";

-                model_weight_token = ModelWeightToken(format!("{}", i));
+                model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                title = format!("SDXL_Niji_Special Edition: {}", i);
                description = format!("Description Number {}:", i);

@@ -263,7 +263,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                "#;

                let lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg";
-                model_weight_token = ModelWeightToken(format!("{}", i));
+                model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                title = format!("Gawr Gura LoRA: {}", i);
                
                description = format!("Description Number {}:", i);
@@ -295,7 +295,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                - Gawr Gura MOS 4
                "#;

-                model_weight_token = ModelWeightToken(format!("{}", i));
+                model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                title = format!("Gawr Gura: {}", i);
                description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i);
                description_rendered_html = rvc_markdown_description;
@@ -356,11 +356,13 @@ pub async fn seed_weights_for_user_token(
 ) -> AnyhowResult<()> {
    info!("Seeding weights...");

-    let model_weight_token1 = ModelWeightToken("11".to_string());
-    let model_weight_token2 = ModelWeightToken("22".to_string());
-    let model_weight_token3 = ModelWeightToken("33".to_string());
-    let model_weight_token4 = ModelWeightToken("44".to_string());
-    let model_weight_token5 = ModelWeightToken("55".to_string());
+    ModelWeightToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(54321);
+
+    let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();

    let model_weights_args = vec![
        CreateModelWeightsArgs {
@@ -503,20 +505,20 @@ pub async fn seed_weights_for_user_token(


 pub async fn original_seed_weights(mysql_pool: &Pool<MySql>,user_token: UserToken) -> AnyhowResult<()> {
-    let model_weight_token1 = ModelWeightToken("1".to_string());
-    let creator_token1 = UserToken("creatorToken1".to_string());
+    let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let creator_token1 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();

-    let model_weight_token2 = ModelWeightToken("2".to_string());
-    let creator_token2 = UserToken("creatorToken2".to_string());
+    let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let creator_token2 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();

-    let model_weight_token3 = ModelWeightToken("3".to_string());
-    let creator_token3 = UserToken("creatorToken3".to_string());
+    let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let creator_token3 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();

-    let model_weight_token4 = ModelWeightToken("4".to_string());
-    let creator_token4 = UserToken("creatorToken4".to_string());
+    let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let creator_token4 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();

-    let model_weight_token5 = ModelWeightToken("5".to_string());
-    let creator_token5 = UserToken("creatorToken5".to_string());
+    let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let creator_token5 = UserToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();

    let model_weights_args = vec![
        CreateModelWeightsArgs {
@@ -0,0 +1,33 @@
+from types import Optional
+
+class FileInfo:
+    size_in_bytes: int
+    mime_type: str
+    file_name: str
+    
+class BucketDetails:
+    bucket_hash: str
+    maybe_bucket_name: Optional[str]
+    maybe_bucket_extension: Optional[str]
+    
+    
+class MediaFileBucketDirectory: 
+  object_hash: str
+  directory: str
+  optional_prefix: Optional[str]
+  optional_prefix_extension: Optional[str]
+  original_file_base_name: str
+  
+class WeightsType:
+    directory: float
+    
+class FileOperationManager:
+    def __init__(self, bucket,bucket_info:BucketDetails):
+        self.bucket = bucket
+        self.bucket_info = bucket_info
+
+    def check_if_file_exists_in_db(self, file_token: str, callback) -> bool:
+        pass
+    
+    
+    
@@ -1,35 +1,29 @@
-use std::path::Path;
-
-use log::{ info, warn };
-use sqlx::{ MySql, Pool };
-
-use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self }, private };
+use std::{path::Path, string};
+use errors::{anyhow, AnyhowResult};
+use log::{info, warn};
+use rand::Rng;
+use sqlx::{MySql, Pool};

+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+use enums::{
+    by_table::model_weights::{weights_category::WeightsCategory, weights_types::WeightsType},
+    common::visibility::Visibility,
+};
 use filesys::file_read_bytes::file_read_bytes;
 use filesys::file_size::file_size;
 use filesys::path_to_string::path_to_string;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
-use mimetypes::mimetype_for_file::get_mimetype_for_file;
-
-use storyteller_root::get_seed_tool_data_root;
-use tokens::tokens::users::UserToken;
-use crate::bucket_clients::BucketClients;
-use crate::seeding::users::HANASHI_USER_TOKEN;
-
-use enums::{
-    by_table::model_weights::{ weights_category::WeightsCategory, weights_types::WeightsType },
-    common::visibility::Visibility,
-};
-
-use rand::Rng;
-use errors::{ anyhow, AnyhowResult };
 use mysql_queries::queries::model_weights::create_weight::{
    create_weight,
    CreateModelWeightsArgs,
-    self,
 };
+use storyteller_root::get_seed_tool_data_root;
 use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::bucket_clients::BucketClients;
+use crate::seeding::users::HANASHI_USER_TOKEN;

 pub async fn seed_weights_files(
    mysql_pool: &Pool<MySql>,
@@ -37,6 +31,9 @@ pub async fn seed_weights_files(
 ) -> AnyhowResult<()> {
    info!("Seeding weights files...");

+    // NB: Stable token generation.
+    ModelWeightToken::reset_rng_for_testing_and_dev_seeding_never_use_in_production_seriously(123);
+
    let seeded_weights = [
        (
            HANASHI_USER_TOKEN,
@@ -87,9 +84,8 @@ pub async fn seed_weights_files(
        let mut rng = rand::thread_rng();
        let n: u32 = rng.gen();

-        let model_weight_token = ModelWeightToken(format!("{}", n)); // random token
-        let model_weight_token = model_weight_token;
-        
+        let model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+
        let title = model_name;
        let description = "This is a description";
        let description_rendered_html = "This is a description rendered html";
@@ -97,9 +93,9 @@ pub async fn seed_weights_files(
        let original_download_url = "https://example.com/model.safetensors";
        let thumbnail_token = format!("{}", n);

-        let public_bucket_hash: String = weights_file_token.to_string();
-        let public_bucket_prefix = weights_type.to_str();
-        let public_bucket_extension = weights_category.to_str();
+        // let public_bucket_hash: String = weights_file_token.to_string();
+        // let public_bucket_prefix = weights_type.to_string();
+        // let public_bucket_extension = weights_category.to_string();

        let cached_user_ratings_total_count = n.clone();
        let cached_user_ratings_positive_count = n.clone();
@@ -114,7 +110,7 @@ pub async fn seed_weights_files(
        let file_size_bytes = file_size(weight_file_path.clone())?;
        let sha256_checksum = sha256_hash_file(weight_file_path.clone())?;
        
-        let result = seed_model(
+        let bucket_details = seed_model(
            &mysql_pool,
            &model_weight_token,
            &user_token,
@@ -122,11 +118,16 @@ pub async fn seed_weights_files(
            &weight_file_path,
            maybe_bucket_clients
        ).await;
+        let bucket_details_data = bucket_details.unwrap();
+
+        let bucket_hash = bucket_details_data.bucket_hash;
+        let maybe_bucket_prefix = bucket_details_data.maybe_bucket_prefix.as_deref().unwrap();
+        let maybe_bucket_extension = bucket_details_data.maybe_bucket_extension.as_deref().unwrap();

        let create_model_weights_args = CreateModelWeightsArgs {
            token: &model_weight_token,
-            weights_type: weights_type, // Assuming weights_type is defined elsewhere
-            weights_category: weights_category, // Assuming weights_category is defined elsewhere
+            weights_type, // Assuming weights_type is defined elsewhere
+            weights_category, // Assuming weights_category is defined elsewhere
            title: title.to_string(),
            maybe_thumbnail_token: Some(thumbnail_token),
            description_markdown: description.to_string(),
@@ -139,37 +140,28 @@ pub async fn seed_weights_files(
            original_filename: Some(original_filename.to_string()),
            file_size_bytes: file_size_bytes as i32, // Assuming the file size is 0
            file_checksum_sha2: sha256_checksum.to_string(), // Assuming the file checksum is an empty string
-            private_bucket_hash: public_bucket_hash,
-            maybe_private_bucket_prefix: Some(public_bucket_prefix.to_string()),
-            maybe_private_bucket_extension: Some(public_bucket_extension.to_string()),
-            cached_user_ratings_total_count: cached_user_ratings_total_count,
-            cached_user_ratings_positive_count: cached_user_ratings_positive_count,
-            cached_user_ratings_negative_count: cached_user_ratings_negative_count,
+            private_bucket_hash: bucket_hash,
+            maybe_private_bucket_prefix: Some(maybe_bucket_prefix.to_string()),
+            maybe_private_bucket_extension: Some(maybe_bucket_extension.to_string()),
+            cached_user_ratings_total_count,
+            cached_user_ratings_positive_count,
+            cached_user_ratings_negative_count,
            maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio),
-            version: version,
+            version,
            mysql_pool: &mysql_pool, // Assuming mysql_pool is defined elsewhere
        };

        create_weight(create_model_weights_args).await?;

-        match result {
-            Ok(_) => info!("Seeded {:?}", weight_file_path),
-            Err(err) =>
-                warn!(
-                    r#"
-              Could not seed media file {} , {:?} : {:?}
-              (No worries: if there's a duplicate key error, we probably already
-              seeded the media file on a previous invocation!)
-            "#,
-                    model_weight_token,
-                    subdirectory_path,
-                    err
-                ),
-        }
+       
+        info!("Seeded {:?}", weight_file_path)
+        
    }

    Ok(())
 }
+
+#[derive(Clone)]
 struct BucketDetails {
    bucket_hash: String,
    maybe_bucket_prefix: Option<String>,
@@ -183,37 +175,33 @@ async fn seed_model(
    weight_type: WeightsType,
    weight_file_path: &Path,
    maybe_bucket_clients: Option<&BucketClients>
-) -> AnyhowResult<()> {
+) -> AnyhowResult<BucketDetails> {
    info!("Seeding Media file {:?} ...", weight_file_path);
    // mutable variables
-    let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
-    let mut maybe_bucket_prefix = "";
-    let mut maybe_bucket_extension = "";
-
    if let Some(bucket_clients) = maybe_bucket_clients {
-        let bucket_details = seed_file_to_bucket(weight_file_path, bucket_clients).await?;
-        bucket_hash = bucket_details.bucket_hash;
-        maybe_bucket_prefix = bucket_details.maybe_bucket_prefix.as_deref().unwrap();
-        maybe_bucket_extension = bucket_details.maybe_bucket_extension.as_deref().unwrap();
+        let bucket_details = seed_file_to_bucket(weight_file_path, bucket_clients,weight_type).await?;
+        return Ok(bucket_details);
    }
-   
-    Ok(())
+    Err(anyhow!("No bucket clients provided!"))
+
 }

 async fn seed_file_to_bucket(
    weight_file_path: &Path,
-    bucket_clients: &BucketClients
+    bucket_clients: &BucketClients,
+    weight_type: WeightsType,
 ) -> AnyhowResult<BucketDetails> {
    info!("Uploading weights file {:?} ...", weight_file_path);

-    let maybe_bucket_prefix = Some("fakeyou_");
+    let maybe_bucket_prefix = Some(format!("{}_",weight_type.to_str()));
+    let maybe_bucket_prefix = maybe_bucket_prefix.as_deref();

    let maybe_bucket_extension = weight_file_path
        .extension()
        .map(|extension| extension.to_str())
        .flatten();
    // we should just have this be file bucket path and then we can use a file descriptor to turn it into a specific type of path
-    let bucket_location = MediaFileBucketPath::generate_new(
+    let bucket_location = WeightFileBucketPath::generate_new(
        maybe_bucket_prefix,
        maybe_bucket_extension
    );
@@ -225,7 +213,7 @@ async fn seed_file_to_bucket(
    let bytes = file_read_bytes(weight_file_path)?;
    let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream");

-    info!("Uploading weights file to bucket path: {:?}", bucket_path);
+    info!("Copy this line here! Uploading weights file to bucket path: {:?}", bucket_path);

    let _r = bucket_clients.public.upload_file_with_content_type(
        &bucket_path,
@@ -29,7 +29,7 @@ tokens = { path = "../../_types/tokens" }
 my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }

 # External
-clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+clap = "4.4.8"
 log = "0.4.14"

 # Runtime
@@ -0,0 +1,24 @@
+use std::path::PathBuf;
+
+use clap::Parser;
+
+use errors::AnyhowResult;
+
+#[derive(Parser, Debug)]
+#[command(name="dev-upload-media-file")]
+pub struct RawArgs {
+  #[arg(name="file", short='f', long="file", help="File to upload", required=true)]
+  pub file_path: String,
+}
+
+pub struct CliArgs {
+  pub file_path: PathBuf,
+}
+
+pub fn parse_cli_args() -> AnyhowResult<CliArgs> {
+  let args = RawArgs::parse();
+
+  Ok(CliArgs {
+    file_path: PathBuf::from(args.file_path),
+  })
+}
@@ -1,28 +0,0 @@
-use clap::{App, Arg};
-
-use errors::AnyhowResult;
-
-pub struct CliArgs {
-  pub seed_cloud_bucket: bool,
-  pub seed_elasticsearch: bool,
-}
-
-pub fn parse_cli_args() -> AnyhowResult<CliArgs> {
-  let matches = App::new("dev-database-seed")
-      .arg(Arg::with_name("seed_cloud_bucket")
-          .long("bucket")
-          .help("Seed the cloud bucket with files")
-          .takes_value(false)
-          .required(false))
-      .arg(Arg::with_name("seed_elasticsearch")
-        .long("elasticsearch")
-        .help("Seed the local elasticsearch")
-        .takes_value(false)
-        .required(false))
-      .get_matches();
-
-  Ok(CliArgs {
-    seed_cloud_bucket: matches.is_present("seed_cloud_bucket"),
-    seed_elasticsearch: matches.is_present("seed_elasticsearch"),
-  })
-}
@@ -1,27 +0,0 @@
-use std::path::PathBuf;
-use clap::{App, Arg};
-use errors::{anyhow, AnyhowResult};
-
-pub struct CliArgs {
-  pub file_path: PathBuf,
-}
-
-pub fn parse_cli_args() -> AnyhowResult<CliArgs> {
-  let matches = App::new("dev_upload_media")
-      .arg(Arg::with_name("file")
-          .short("f")
-          .long("file")
-          .value_name("FILE")
-          .help("File to upload")
-          .takes_value(true)
-          .required(true))
-      .get_matches();
-
-  let file_path = matches.value_of("file")
-      .map(|val| val.trim().to_string())
-      .ok_or(anyhow!("no file path supplied"))?;
-
-  Ok(CliArgs {
-    file_path: PathBuf::from(file_path),
-  })
-}
@@ -29,7 +29,7 @@ my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }

 # External
 log = "0.4.14"
-clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+clap = "4.4.8"
 strum = { version = "0.25", features = ["derive"] }

 # Runtime
@@ -1,12 +1,12 @@
 use std::str::FromStr;

-use clap::{App, Arg, ArgMatches};
+use clap::Parser;
 use strum::{EnumCount, EnumString, IntoEnumIterator};
 use strum::EnumIter;

 use errors::{anyhow, AnyhowResult, bail};

-pub struct CliArgs {
+pub struct ParsedArgs {
  pub mysql_environment: Environment,
  pub elasticsearch_environment: Environment,
  pub action: Action,
@@ -25,53 +25,38 @@ pub enum Action {
  SearchTts,
 }

-pub fn parse_cli_args() -> AnyhowResult<CliArgs> {
-  let matches = App::new("dev-database-seed")
-      .arg(Arg::with_name("action")
-          .long("action")
-          .help("action to take")
-          .takes_value(true)
-          .required(true))
-      .arg(Arg::with_name("mysql")
-          .long("mysql")
-          .help("Production or development")
-          .takes_value(true)
-          .required(false))
-      .arg(Arg::with_name("elasticsearch")
-        .long("elasticsearch")
-        .help("Production or development")
-        .takes_value(true)
-        .required(false))
-      .get_matches();
+#[derive(Parser, Debug)]
+#[command(name="elasticsearch-cli")]
+pub struct Args {
+  #[arg(name="action", long="action", help="action to take", required=true)]
+  action: String,

-  Ok(CliArgs {
-    mysql_environment: to_environment(&matches, "mysql")?,
-    elasticsearch_environment: to_environment(&matches, "elasticsearch")?,
-    action: get_action(&matches)?,
+  #[arg(name="mysql", long="mysql", help="production or development")]
+  mysql: Option<String>,
+
+  #[arg(name="elasticsearch", long="elasticsearch", help="production or development")]
+  elasticsearch: Option<String>,
+}
+
+pub fn parse_cli_args() -> AnyhowResult<ParsedArgs> {
+  let args = Args::parse();
+
+  Ok(ParsedArgs {
+    mysql_environment: to_environment(args.mysql.as_deref())?,
+    elasticsearch_environment: to_environment(args.elasticsearch.as_deref())?,
+    action: action_from_str(&args.action)?,
  })
 }

-fn to_environment(matches: &ArgMatches, key: &str) -> AnyhowResult<Environment> {
-  let value = matches.value_of(key)
-      .map(|s| s.to_lowercase());
-
-  Ok(match value.as_deref() {
+fn to_environment(environment: Option<&str>) -> AnyhowResult<Environment> {
+  Ok(match environment {
    None => Environment::Development,
    Some("dev") | Some("development") => Environment::Development,
    Some("prod") | Some("production") => Environment::Production,
-    _ => bail!("invalid environment: {:?}", value),
+    _ => bail!("invalid environment: {:?}", environment),
  })
 }

-fn get_action(matches: &ArgMatches) -> AnyhowResult<Action> {
-  let action = matches.value_of("action")
-      .map(|s| action_from_str(s))
-      .transpose()?
-      .ok_or(anyhow!("no action provided"))?;
-
-  Ok(action)
-}
-
 fn action_from_str(value: &str) -> AnyhowResult<Action> {
  let action = Action::from_str(value)
      .map_err(|err| {
@@ -21,7 +21,7 @@ logging = { path = "../../lib/logging" }
 my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }

 # External
-clap = { version = "4.4.7", features = ["derive"] }
+clap = { version = "4.4.8", features = ["derive"] }
 walkdir = "2.4.0"

 [dev-dependencies]
@@ -0,0 +1,44 @@
+[package]
+name = "migration-tool"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[[bin]]
+name = "migration-tool"
+path = "src/main.rs"
+
+[dependencies]
+# Internal
+buckets = { path = "../../_types/buckets" }
+config = { path = "../../lib/deprecated/config" }
+easyenv = { path = "../../lib/easyenv" }
+enums = { path = "../../_types/enums" }
+errors = { path = "../../lib/errors" }
+mysql_queries = { path = "../../lib/mysql_queries" }
+storyteller_root = { path = "../../lib/storyteller_root" }
+tokens = { path = "../../_types/tokens" }
+
+# Workspace hack (faster compile times)
+my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
+
+# External
+clap = "4.4.8"
+log = "0.4.14"
+strum = { version = "0.25", features = ["derive"] }
+
+# Async + clients
+tokio = { version = "1.34.0", features = ["macros"] }
+sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+serde_json = "1.0.108"
+
+[dev-dependencies]
+# None yet
@@ -0,0 +1,23 @@
+//! migration-tool
+//!
+//! Migrate database records.
+//!
+
+
+use config::shared_constants::DEFAULT_RUST_LOG;
+use errors::AnyhowResult;
+
+#[tokio::main]
+pub async fn main() -> AnyhowResult<()> {
+  println!("migration-tool: migrate database records");
+
+  easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
+
+  // NB: This secrets file differs from the rest because we might actually want to cross
+  // development/production boundaries for migration. We don't want to pull in secrets
+  // from other sources. (Hopefully this isn't getting out of hand at this point.)
+  easyenv::from_filename(".env-migration-tool-secrets")?;
+
+
+  Ok(())
+}
@@ -22,5 +22,5 @@ web_scrapers = { path = "../../lib/web_scrapers" }
 my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }

 # External
-clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+clap = "4.4.8"
 tokio = { version = "1.34.0", features = ["macros"] }
@@ -1,30 +1,24 @@
-use clap::{App, Arg};
+use clap::Parser;

-use errors::{anyhow, AnyhowResult};
+use errors::AnyhowResult;
 use web_scrapers::scrape_supported_webpage::scrape_supported_webpage;

+#[derive(Parser, Debug)]
+#[command(name = "scrape_test")]
+pub struct Args {
+  #[arg(name="url", short='u', long = "url", help = "URL of the page to scrape", required = true)]
+  url: String,
+
+  #[arg(name="print_html", long = "html", help = "Print the HTML scraped", required = true)]
+  print_html: bool,
+}
+
 #[tokio::main]
 pub async fn main() -> AnyhowResult<()> {
-  let matches = App::new("scrape_test")
-      .arg(Arg::with_name("url")
-          .short("u")
-          .long("url")
-          .value_name("URL")
-          .help("URL of the page to scrape")
-          .takes_value(true)
-          .required(true))
-      .arg(Arg::with_name("print_html")
-          .long("html")
-          .help("Print the HTML scraped")
-          .takes_value(false)
-          .required(false))
-      .get_matches();
+  let args = Args::parse();

-  let url = matches.value_of("url")
-      .map(|val| val.trim().to_string())
-      .ok_or(anyhow!("no url supplied"))?;
-
-  let print_html = matches.is_present("print_html");
+  let url = args.url.trim().to_string();
+  let print_html = args.print_html;

  let scrape_result = scrape_supported_webpage(&url).await?;

@@ -0,0 +1,234 @@
+use chrono::{DateTime, Utc};
+use sqlx::{Acquire, FromRow, MySql, MySqlConnection, MySqlPool, QueryBuilder, Row};
+use sqlx::mysql::MySqlRow;
+
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::traits::mysql_from_row::MySqlFromRow;
+use errors::AnyhowResult;
+use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::users::UserToken;
+
+#[derive(Serialize)]
+pub struct MediaFilesByTokensRecord {
+  pub token: MediaFileToken,
+
+  pub media_type: MediaFileType,
+
+  pub origin_category: MediaFileOriginCategory,
+  pub origin_product_category: MediaFileOriginProductCategory,
+  pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
+  pub maybe_origin_model_token: Option<String>,
+
+  pub public_bucket_directory_hash: String,
+  pub maybe_public_bucket_prefix: Option<String>,
+  pub maybe_public_bucket_extension: Option<String>,
+
+  pub maybe_creator_user_token: Option<UserToken>,
+  pub maybe_creator_username: Option<String>,
+  pub maybe_creator_display_name: Option<String>,
+  pub maybe_creator_email_gravatar_hash: Option<String>,
+
+  pub created_at: DateTime<Utc>,
+  pub updated_at: DateTime<Utc>,
+}
+
+pub async fn list_media_files_by_tokens(
+  mysql_pool: &MySqlPool,
+  media_file_tokens: &[MediaFileToken],
+  can_see_deleted: bool
+) -> AnyhowResult<Vec<MediaFilesByTokensRecord>> {
+
+  let mut connection = mysql_pool.acquire().await?;
+
+  let raw_media_files: Vec<RawMediaFileJoinUser> = get_raw_media_files_by_tokens(&mut connection, media_file_tokens, can_see_deleted).await?;
+
+  Ok(map_to_media_files(raw_media_files))
+}
+
+async fn get_raw_media_files_by_tokens(
+  connection: &mut MySqlConnection,
+  media_file_tokens: &[MediaFileToken],
+  can_see_deleted: bool
+) -> AnyhowResult<Vec<RawMediaFileJoinUser>> {
+
+  let connection = connection.acquire().await?;
+
+  let mut query_builder : QueryBuilder<MySql> = if can_see_deleted {
+    QueryBuilder::new(r#"
+      SELECT
+          m.token,
+          m.media_type,
+
+          m.origin_category,
+          m.origin_product_category,
+          m.maybe_origin_model_type,
+          m.maybe_origin_model_token,
+
+          users.token as maybe_creator_user_token,
+          users.username as maybe_creator_username,
+          users.display_name as maybe_creator_display_name,
+          users.email_gravatar_hash as maybe_creator_email_gravatar_hash,
+
+          m.public_bucket_directory_hash,
+          m.maybe_public_bucket_prefix,
+          m.maybe_public_bucket_extension,
+
+          m.created_at,
+          m.updated_at
+
+      FROM media_files as m
+      LEFT OUTER JOIN users
+          ON users.token = m.maybe_creator_user_token
+      WHERE
+          m.creator_set_visibility = "public"
+          AND m.token IN (
+      "#,
+    )
+
+  } else {
+    QueryBuilder::new(r#"
+      SELECT
+          m.token,
+          m.media_type,
+
+          m.origin_category,
+          m.origin_product_category,
+          m.maybe_origin_model_type,
+          m.maybe_origin_model_token,
+
+          users.token as maybe_creator_user_token,
+          users.username as maybe_creator_username,
+          users.display_name as maybe_creator_display_name,
+          users.email_gravatar_hash as maybe_creator_email_gravatar_hash,
+
+          m.public_bucket_directory_hash,
+          m.maybe_public_bucket_prefix,
+          m.maybe_public_bucket_extension,
+
+          m.created_at,
+          m.updated_at
+
+      FROM media_files as m
+      LEFT OUTER JOIN users
+          ON users.token = m.maybe_creator_user_token
+      WHERE
+          m.creator_set_visibility = "public"
+          AND m.user_deleted_at IS NULL
+          AND m.mod_deleted_at IS NULL
+          AND m.token IN (
+      "#
+    )
+  };
+
+  query_builder.push(token_predicate(media_file_tokens));
+
+  query_builder.push(")");
+
+  let query = query_builder.build_query_as::<RawMediaFileJoinUser>();
+
+  let results = query.fetch_all(connection).await?;
+
+  Ok(results)
+}
+
+/// Return a comma-separated predicate, since SQLx does not yet support WHERE IN(?) for Vec<T>, etc.
+/// Issue: https://github.com/launchbadge/sqlx/issues/875
+fn token_predicate(tokens: &[MediaFileToken]) -> String {
+  tokens.iter()
+      .map(|ty| ty.as_str())
+      .map(|ty| format!("\"{}\"", ty))
+      .collect::<Vec<String>>()
+      .join(", ")
+}
+
+fn map_to_media_files(dataset:Vec<RawMediaFileJoinUser>) -> Vec<MediaFilesByTokensRecord> {
+  let media_files: Vec<MediaFilesByTokensRecord> = dataset
+      .into_iter()
+      .map(|media_file: RawMediaFileJoinUser| {
+        MediaFilesByTokensRecord {
+          token: media_file.token,
+
+          origin_category: media_file.origin_category,
+          origin_product_category: media_file.origin_product_category,
+          maybe_origin_model_type: media_file.maybe_origin_model_type,
+          maybe_origin_model_token: media_file.maybe_origin_model_token,
+
+          maybe_creator_user_token: media_file.maybe_creator_user_token,
+          maybe_creator_username: media_file.maybe_creator_username,
+          maybe_creator_display_name: media_file.maybe_creator_display_name,
+          maybe_creator_email_gravatar_hash: media_file.maybe_creator_email_gravatar_hash,
+
+          media_type: media_file.media_type,
+          public_bucket_directory_hash: media_file.public_bucket_directory_hash,
+          maybe_public_bucket_prefix: media_file.maybe_public_bucket_prefix,
+          maybe_public_bucket_extension: media_file.maybe_public_bucket_extension,
+
+          created_at: media_file.created_at,
+          updated_at: media_file.updated_at,
+        }
+      }).collect();
+
+  media_files
+}
+
+  #[derive(Serialize)]
+  pub struct RawMediaFileJoinUser {
+    pub token: MediaFileToken,
+
+    pub media_type: MediaFileType,
+
+    pub origin_category: MediaFileOriginCategory,
+    pub origin_product_category: MediaFileOriginProductCategory,
+    pub maybe_origin_model_type: Option<MediaFileOriginModelType>,
+    pub maybe_origin_model_token: Option<String>,
+
+    pub maybe_creator_user_token: Option<UserToken>,
+    pub maybe_creator_username: Option<String>,
+    pub maybe_creator_display_name: Option<String>,
+    pub maybe_creator_email_gravatar_hash: Option<String>,
+
+    pub public_bucket_directory_hash: String,
+    pub maybe_public_bucket_prefix: Option<String>,
+    pub maybe_public_bucket_extension: Option<String>,
+
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+}
+
+// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or
+// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually
+// implemented. Perhaps this will improve in the future, but for now manually constructed queries
+// cannot have type hints, eg. the following:
+//
+//    m.token as `token: tokens::tokens::media_files::MediaFileToken`,
+//    m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,
+//    m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
+//
+// This results in the automatic mapping not being able to be found by name (for macro derive), and
+// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the
+// full "as" clause).
+impl FromRow<'_, MySqlRow> for RawMediaFileJoinUser {
+  fn from_row(row: &MySqlRow) -> Result<Self, sqlx::Error> {
+    Ok(Self {
+      token: MediaFileToken::new(row.try_get("token")?),
+      media_type: MediaFileType::try_from_mysql_row(row, "media_type")?,
+      origin_category: MediaFileOriginCategory::try_from_mysql_row(row, "origin_category")?,
+      origin_product_category: MediaFileOriginProductCategory::try_from_mysql_row(row, "origin_product_category")?,
+      maybe_origin_model_type: MediaFileOriginModelType::try_from_mysql_row_nullable(row, "maybe_origin_model_type")?,
+      maybe_origin_model_token: row.try_get("maybe_origin_model_token")?,
+      maybe_creator_user_token: row.try_get::<Option<String>, _>("maybe_creator_user_token")?
+          .and_then(|token| Some(UserToken::new_from_str(&token))),
+      maybe_creator_username: row.try_get("maybe_creator_username")?,
+      maybe_creator_display_name: row.try_get("maybe_creator_display_name")?,
+      maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?,
+      public_bucket_directory_hash: row.try_get("public_bucket_directory_hash")?,
+      maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?,
+      maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?,
+      created_at: row.try_get("created_at")?,
+      updated_at: row.try_get("updated_at")?,
+    })
+  }
+}
@@ -16,11 +16,8 @@ pub struct MediaFileListPage {

  pub sort_ascending: bool,

-  /// ID of the first record.
-  pub first_id: Option<i64>,
-
-  /// ID of the last record.
-  pub last_id: Option<i64>,
+  pub current_page: usize,
+  pub total_page_count: usize,
 }

 pub struct MediaFileListItem {
@@ -51,9 +48,9 @@ pub enum ViewAs {

 pub struct ListMediaFileForUserArgs<'a> {
  pub username: &'a str,
-  pub limit: usize,
  pub maybe_filter_media_type: Option<MediaFileType>,
-  pub maybe_offset: Option<usize>,
+  pub page_size: usize,
+  pub page_index: usize,
  pub cursor_is_reversed: bool,
  pub view_as: ViewAs,
  pub mysql_pool: &'a MySqlPool,
@@ -61,25 +58,42 @@ pub struct ListMediaFileForUserArgs<'a> {

 pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> AnyhowResult<MediaFileListPage> {

+  /// Let's figure out how many results we could have returned total
+  let count_fields = select_total_count_field();
+  let mut count_query_builder = query_builder(
+    args.maybe_filter_media_type,
+    args.username,
+    false,
+    0,
+    0,
+    args.cursor_is_reversed,
+    args.view_as,
+    count_fields.as_str(),
+  );
+
+  let row_count_query = count_query_builder.build_query_scalar::<i64>();
+  let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?;
+  /// Figure out limit start and end based on page size and indexes as requested
+  let limit_start = args.page_size * args.page_index;
+  let limit_end = limit_start + args.page_size;
+
+  /// Now fetch the actual results with all the fields
+  let result_fields = select_result_fields();
  let mut query = query_builder(
    args.maybe_filter_media_type,
    args.username,
-    args.limit,
-    args.maybe_offset,
+    true,
+    limit_start,
+    limit_end,
    args.cursor_is_reversed,
    args.view_as,
+    result_fields.as_str(),
  );

  let query = query.build_query_as::<MediaFileListItemInternal>();
-
  let results = query.fetch_all(args.mysql_pool).await?;

-  let first_id = results.first()
-      .map(|raw_result| raw_result.id);
-
-  let last_id = results.last()
-      .map(|raw_result| raw_result.id);
-
+  let number_of_pages = (row_count_result / args.page_size as i64) as usize;
  let results = results.into_iter()
      .map(|record| {
        MediaFileListItem {
@@ -102,43 +116,59 @@ pub async fn list_media_files_for_user(args: ListMediaFileForUserArgs<'_>) -> An
  Ok(MediaFileListPage {
    records: results,
    sort_ascending: !args.cursor_is_reversed,
-    first_id,
-    last_id,
+    current_page: args.page_index,
+    total_page_count: number_of_pages,
  })
 }

+
+fn select_result_fields() -> String {
+  r#"
+    m.id,
+    m.token,
+
+    m.origin_category,
+    m.origin_product_category,
+
+    m.maybe_origin_model_type,
+    m.maybe_origin_model_token,
+
+    m.media_type,
+
+    m.public_bucket_directory_hash,
+    m.maybe_public_bucket_prefix,
+    m.maybe_public_bucket_extension,
+
+    m.creator_set_visibility,
+    m.created_at,
+    m.updated_at
+  "#
+  .to_string()
+}
+
+fn select_total_count_field() -> String {
+  r#"
+    COUNT(m.id) AS total_count
+  "#
+  .to_string()
+}
+
 fn query_builder<'a>(
  maybe_filter_media_type: Option<MediaFileType>,
  username: &'a str,
-  limit: usize,
-  maybe_offset: Option<usize>,
+  enforce_limits: bool,
+  limit_start: usize,
+  limit_end: usize,
  cursor_is_reversed: bool,
  view_as: ViewAs,
+  select_fields: &'a str,
 ) -> QueryBuilder<'a, MySql> {

  // NB: Query cannot be statically checked by sqlx
  let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
-    r#"
+    format!(r#"
 SELECT
-  m.id,
-  m.token,
-
-  m.origin_category,
-  m.origin_product_category,
-
-  m.maybe_origin_model_type,
-  m.maybe_origin_model_token,
-
-  m.media_type,
-
-  m.public_bucket_directory_hash,
-  m.maybe_public_bucket_prefix,
-  m.maybe_public_bucket_extension,
-
-  m.creator_set_visibility,
-  m.created_at,
-  m.updated_at
-
+     {select_fields}
 FROM media_files AS m
 LEFT OUTER JOIN users AS u
    ON m.maybe_creator_user_token = u.token
@@ -146,7 +176,7 @@ LEFT OUTER JOIN users AS u
 WHERE m.user_deleted_at IS NULL
  AND m.mod_deleted_at IS NULL
    "#
-  );
+  ));

  query_builder.push(" AND u.username = ");
  query_builder.push_bind(username);
@@ -171,21 +201,19 @@ WHERE m.user_deleted_at IS NULL
    }
  }

-  if let Some(offset) = maybe_offset {
-    if cursor_is_reversed {
-      query_builder.push(format!(" AND m.id < {offset} "));
-    } else {
-      query_builder.push(format!(" AND m.id > {offset} "));
-    }
-  }
-
  if cursor_is_reversed {
    query_builder.push(" ORDER BY m.id ASC ");
  } else {
    query_builder.push(" ORDER BY m.id DESC ");
  }

-  query_builder.push(format!(" LIMIT {limit} "));
+  if enforce_limits {
+    if cursor_is_reversed {
+      query_builder.push(format!(" LIMIT {limit_start}, {limit_end} "));
+    } else {
+      query_builder.push(format!(" LIMIT {limit_end}, {limit_start} "));
+    }
+  }

  query_builder
 }
@@ -2,10 +2,11 @@ pub mod delete_media_file;
 pub mod get_media_file;
 pub mod insert_media_file_from_cli_tool;
 pub mod insert_media_file_from_face_animation;
+pub mod insert_media_file_from_rerender;
 pub mod insert_media_file_from_tacotron2;
 pub mod insert_media_file_from_voice_conversion;
 pub mod insert_media_file_from_zero_shot_tts;
 pub mod list_media_files;
+pub mod list_media_files_by_tokens;
 pub mod list_media_files_for_user;
 pub mod update_media_file;
-pub mod insert_media_file_from_rerender;
@@ -47,7 +47,6 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
 # External
 anyhow = "1.0.75"
 chrono = { version = "0.4.22", features = ["serde"] }
-clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
 dotenv = "0.15.0"
 hostname = "0.3.1"
 lfu = "0.2.5"
@@ -1,11 +1,4 @@
-use anyhow::anyhow;
-use log::info;
-
-use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr;
-use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::media_files::get_media_file::get_media_file;
-use tokens::tokens::media_files::MediaFileToken;

 use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -33,7 +33,6 @@ my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
 # External
 anyhow = "1.0.75"
 chrono = { version = "0.4.22", features = ["serde"] }
-clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
 dotenv = "0.15.0"
 hostname = "0.3.1"
 lfu = "0.2.5"
@@ -18,7 +18,6 @@
 use std::path::PathBuf;
 use std::time::Duration;

-use clap::{App, Arg};
 use log::{error, info, warn};
 use r2d2_redis::r2d2;
 use r2d2_redis::RedisConnectionManager;
@@ -93,15 +92,6 @@ async fn main() -> AnyhowResult<()> {
  // This file should only contain *development* secrets, never production.
  let _ = dotenv::from_filename(".env-secrets").ok();

-  let matches = App::new("tts-inference-job")
-      .arg(Arg::with_name("sidecar_hostname")
-          .long("sidecar_hostname")
-          .value_name("HOSTNAME")
-          .help("Hostname for the TTS inference sidecar")
-          .takes_value(true)
-          .required(false))
-      .get_matches();
-
  info!("Obtaining worker hostname...");

  let server_hostname = hostname::get()
@@ -150,13 +140,9 @@ async fn main() -> AnyhowResult<()> {
    Some(bucket_timeout),
  )?;

-  let mut sidecar_hostname =
+  let sidecar_hostname =
      easyenv::get_env_string_required(ENV_TTS_INFERENCE_SIDECAR_HOSTNAME)?;

-  if let Some(hostname) = matches.value_of("sidecar_hostname") {
-    sidecar_hostname = hostname.to_string();
-  }
-
  info!("Sidecar hostname: {:?}", sidecar_hostname);

  let tts_inference_sidecar_client =
@@ -20,6 +20,7 @@ http_server_common = { path = "../../../lib/deprecated/http_server_common" }
 mysql_queries = { path = "../../../lib/mysql_queries" }
 reusable_types = { path = "../../../_types/reusable_types" }
 url_config = { path = "../../../lib/url_config" }
+user_traits_component = { path = "../user_traits_component" }

 # Workspace hack (faster compile times)
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -45,7 +46,7 @@ serde_json = "1.0.108"
 toml = "0.5.8"

 # Money
-async-stripe = { version = "0.25.2", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] }
+async-stripe = { version = "0.26.0", default-features = false, features = ["runtime-tokio-hyper", "connect", "billing", "checkout", "webhook-events"] }

 # Async traits
 # NB: note: `async` trait functions are not currently supported
@@ -3,12 +3,12 @@ use actix_web::web::Json;

 use reusable_types::server_environment::ServerEnvironment;
 use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
+use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge;

 use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError;
 use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_shared::stripe_create_checkout_session_shared;
 use crate::stripe::stripe_config::StripeConfig;
 use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup;
-use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge;
 use crate::stripe::traits::internal_user_lookup::InternalUserLookup;

 // =============== Request ===============
@@ -4,12 +4,12 @@ use actix_web::web::Query;

 use reusable_types::server_environment::ServerEnvironment;
 use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
+use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge;

 use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError;
 use crate::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_shared::stripe_create_checkout_session_shared;
 use crate::stripe::stripe_config::StripeConfig;
 use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup;
-use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge;
 use crate::stripe::traits::internal_user_lookup::InternalUserLookup;

 // =============== Request ===============
@@ -2,12 +2,12 @@ use actix_web::{HttpRequest, HttpResponse, web};
 use actix_web::web::Json;

 use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
+use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge;

 use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_error::CreateCustomerPortalSessionError;
 use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_shared::stripe_create_customer_portal_session_shared;
 use crate::stripe::stripe_config::StripeConfig;
 use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup;
-use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge;
 use crate::stripe::traits::internal_user_lookup::InternalUserLookup;

 // =============== Request ===============
@@ -3,12 +3,12 @@ use actix_web::{HttpRequest, HttpResponse, web};
 use actix_web::web::Query;

 use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
+use user_traits_component::traits::internal_session_cache_purge::InternalSessionCachePurge;

 use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_error::CreateCustomerPortalSessionError;
 use crate::stripe::http_endpoints::customer_portal::stripe_create_customer_portal_session_shared::stripe_create_customer_portal_session_shared;
 use crate::stripe::stripe_config::StripeConfig;
 use crate::stripe::traits::internal_product_to_stripe_lookup::InternalProductToStripeLookup;
-use crate::stripe::traits::internal_session_cache_purge::InternalSessionCachePurge;
 use crate::stripe::traits::internal_user_lookup::InternalUserLookup;

 // =============== Request ===============
@@ -1,4 +1,3 @@
 pub mod internal_product_to_stripe_lookup;
-pub mod internal_session_cache_purge;
 pub mod internal_subscription_product_lookup;
 pub mod internal_user_lookup;
@@ -0,0 +1,27 @@
+[package]
+publish = false
+name = "user_traits_component"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+
+[lib]
+name = "user_traits_component"
+path = "src/lib.rs"
+
+[dependencies]
+
+# Internal
+errors = { path = "../../../lib/errors" }
+
+# Workspace hack (faster compile times)
+my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+
+# Web
+actix-web = "4.4.0"
+
+[dev-dependencies]
+mockall = "0.11.3"
@@ -0,0 +1,21 @@
+//! user_traits_component
+//!
+//! Expose some traits for other plugins / components.
+//! This will probably go away once we start consolidating the plugins into the main service.
+//!
+
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![allow(unused_variables)] // NB: issue with automock
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod traits;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Michael Chung	87a7bd6ac4	Seed Model weights fix	2023-12-18 21:08:43 -05:00
Brandon Thomas	0578980205	re-home nessu's models (per vegito)	2023-12-16 23:26:16 -05:00
Brandon Thomas	732dd27d0f	fix build errors; kebab-case all CLI apps	2023-12-15 19:39:06 -05:00
Brandon Thomas	c137e1ace3	migration tool binary bootstrap	2023-12-15 19:28:37 -05:00
ArEnSc	f31e93b109	Merge pull request #218 from storytold/feature/weights_files Feature/weights files	2023-12-15 18:36:31 -05:00
ArEnSc	6aeb325d10	Merge branch 'master' into feature/weights_files	2023-12-15 18:36:23 -05:00
Michael Chung	2c6a07659a	Ok weights now are properly linked.	2023-12-15 16:59:13 -05:00
Michael Chung	0e1f6187bf	Weights path working on the generic version.	2023-12-15 16:29:36 -05:00
Brandon Thomas	a6806d3616	indices on model_weights and users (there weren't any on model_weights!)	2023-12-15 13:57:22 -05:00
Brandon Thomas	98fa0a67ea	rename extension tables	2023-12-15 13:33:14 -05:00
Brandon Thomas	11e3a4e429	rename `model_weights_samples` -> `model_weights_preview_samples` - doing this before anyone else runs these migrations - also added "squashed" documentation on the model_weights and associated tables - also added (commented) good indices for model_weights	2023-12-15 12:47:33 -05:00
Brandon Thomas	d9e45c0291	Merge pull request #217 from storytold/brandon-lots-of-schema-changes-afoot Lots of schema changes afoot - model_weights migration, covers, samples	2023-12-15 10:27:58 -05:00
Brandon Thomas	5108f94c0f	Lots of schema changes afoot - model_weights migration, covers, samples Model weights migration (old tokens): - model_weights.maybe_migration_old_model_token Model weights migration (new tokens): - tts_models.maybe_migration_new_model_weights_token - vocoder_models.maybe_migration_new_model_weights_token - voice_conversion_models.maybe_migration_new_model_weights_token Cover and banner images for models: - model_weights.maybe_banner_media_file_token - model_weights.maybe_cover_media_file_token Avatar and banner images for users: - users.maybe_avatar_media_file_token - users.maybe_cover_media_file_token Sample multimedia for models: - model_weights_samples (new table)	2023-12-15 10:26:08 -05:00
Brandon Thomas	6fc1dd263c	migration dir had a space in it	2023-12-14 20:23:03 -05:00
Brandon Thomas	186979ecbd	fix lints and other issues	2023-12-14 18:40:44 -05:00
Brandon Thomas	6c9b0a5c2a	fix bad github merge	2023-12-14 18:30:54 -05:00
Brandon Thomas	92b5a4c90b	Merge pull request #215 from storytold/brandon-fix-clap-upgrade Fix clap in various CLI tools post 2.x -> 4.x upgrade	2023-12-14 18:23:46 -05:00
Brandon Thomas	e9cd505e0b	Fix clap in various CLI tools post 2.x -> 4.x upgrade	2023-12-14 18:22:05 -05:00
Brandon Thomas	561493182e	Merge pull request #164 from storytold/dependabot/cargo/clap-4.4.8 Bump clap from 2.34.0 to 4.4.8	2023-12-14 17:52:26 -05:00
Brandon Thomas	5f673095eb	Merge pull request #173 from storytold/dependabot/cargo/old_code/pubsub-enricher/openssl-0.10.60 Bump openssl from 0.10.33 to 0.10.60 in /old_code/pubsub-enricher	2023-12-14 17:36:43 -05:00
Brandon Thomas	bf3c199592	Merge pull request #172 from storytold/dependabot/cargo/old_code/twitch-ingestion-client/openssl-0.10.60 Bump openssl from 0.10.32 to 0.10.60 in /old_code/twitch-ingestion-client	2023-12-14 17:36:19 -05:00
Brandon Thomas	9cf629c324	Merge pull request #174 from storytold/dependabot/cargo/old_code/twitter-ingestion-client/openssl-0.10.60 Bump openssl from 0.10.34 to 0.10.60 in /old_code/twitter-ingestion-client	2023-12-14 17:36:11 -05:00
Brandon Thomas	4a6009cf78	Merge pull request #175 from storytold/dependabot/cargo/old_code/twitch-tts-desktop-client/openssl-0.10.60 Bump openssl from 0.10.36 to 0.10.60 in /old_code/twitch-tts-desktop-client	2023-12-14 17:36:01 -05:00
Brandon Thomas	8789c53a0a	Merge pull request #183 from storytold/dependabot/cargo/async-stripe-0.26.0 Bump async-stripe from 0.25.2 to 0.26.0	2023-12-14 17:27:07 -05:00
Brandon Thomas	46f5689cbb	Merge pull request #213 from storytold/dec-14-mithoo make media endpoint for profiles follow public pagination	2023-12-14 17:26:13 -05:00
Brandon Thomas	97ee848e64	Merge pull request #178 from storytold/nov-29-mithoo update media file visibilty handler	2023-12-14 17:23:53 -05:00
Brandon Thomas	2f67bd41cf	Merge branch 'master' into nov-29-mithoo	2023-12-14 17:23:45 -05:00
Brandon Thomas	4b2cfaeb09	Merge pull request #211 from storytold/brandon-deterministic-test-tokens deterministic test tokens	2023-12-14 17:03:22 -05:00
Brandon Thomas	b05cde1de1	Merge pull request #214 from storytold/brandon-fix-logout fix logout	2023-12-14 17:01:55 -05:00
Brandon Thomas	60d4199216	fix logout The reason this is so labyrinthine is because of the users and billing components. They bind dynamic behaviors that are defined in storyteller-web (one of them being clearing the redis session cache). Actually, the users component _didn't_, because it was defined in the billing component. So I hoisted it out into a new component library. In the fullness of time I imagine all of this will be consolidated back under storyteller-web so we don't have to worry about binding actions at a distance. Anyway, this is tested and works locally.	2023-12-14 16:58:37 -05:00
Brandon Thomas	a88d888c18	featured media weights endpoint	2023-12-14 16:35:07 -05:00
Brandon Thomas	5a4d2a8c06	media files query by tokens (WHERE IN clause)	2023-12-14 15:15:31 -05:00
Brandon Thomas	32b1786b59	No hateful / antisemitic models on our service (hitler)	2023-12-14 14:53:17 -05:00
Brandon Thomas	8618e774e7	codegen db	2023-12-14 13:37:25 -05:00
kasisnu	ff7ea2a4db	make media endpoint for profiles follow public pagination	2023-12-14 16:27:21 +05:30
Brandon Thomas	b85415c03a	Merge pull request #212 from storytold/brandon-polymorphic-token-checks polymorphic token checks (thanks kasisnu!) + new token types	2023-12-14 04:40:53 -05:00
Brandon Thomas	d447ad7d86	deterministic test tokens it'd be good to use the same token values in tests, dev seed tools, etc. this should be a threadsafe, resettable, deterministic RNG	2023-12-14 03:49:58 -05:00
dependabot[bot]	cf034a900e	Bump clap from 2.34.0 to 4.4.8 Bumps [clap](https://github.com/clap-rs/clap) from 2.34.0 to 4.4.8. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v2.34.0...v4.4.8) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com>	2023-12-14 04:27:36 +00:00
dependabot[bot]	8bb82c0fe5	Bump async-stripe from 0.25.2 to 0.26.0 Bumps [async-stripe](https://github.com/arlyon/async-stripe) from 0.25.2 to 0.26.0. - [Release notes](https://github.com/arlyon/async-stripe/releases) - [Changelog](https://github.com/arlyon/async-stripe/blob/master/CHANGELOG.md) - [Commits](https://github.com/arlyon/async-stripe/compare/v0.25.2...v0.26.0) --- updated-dependencies: - dependency-name: async-stripe dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-12-12 16:59:57 +00:00
kasisnu	133ccff348	update media file visibilty handler	2023-11-29 17:12:22 +05:30
dependabot[bot]	a3ab634725	Bump openssl in /old_code/twitch-tts-desktop-client Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.36 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.36...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-28 21:11:08 +00:00
dependabot[bot]	93ebe101f3	Bump openssl in /old_code/twitter-ingestion-client Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.34 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.34...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-28 21:03:53 +00:00
dependabot[bot]	0128134d3d	Bump openssl from 0.10.33 to 0.10.60 in /old_code/pubsub-enricher Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.33 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.33...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-28 21:02:17 +00:00
dependabot[bot]	da15968c4e	Bump openssl in /old_code/twitch-ingestion-client Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.32 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.32...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-28 21:00:56 +00:00