remove excess message

bvh result type is video; mime type/extension is set correctly

.env

@@ -78,4 +78,5 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt
 # Development key. Permissible to check in.
 NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL
 
-STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting.
+# DO NOT UNCOMMENT THIS :P :P :P
+#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting.

.gitignore

@@ -17,6 +17,7 @@ production_database_voice_clone_report_download.sh
 secrets.toml
 secrets.yaml
 twitch_secrets.toml
+.DS_Store
 
 # Old env secrets
 .*-secrets

.sqlx/query-0220f288c06686874df87ec163533be64cc000634920d58e7fa7a7d156cbfcae.json

@@ -157,7 +157,7 @@
         "ordinal": 15,
         "name": "file_size_bytes",
         "type_info": {
-          "type": "Long",
+          "type": "LongLong",
           "flags": "NOT_NULL",
           "char_set": 63,
           "max_size": 10

.sqlx/query-423109f2dcee23a558c43ecaab0eb2836714d5316620daaa1127b447cad59212.json

@@ -1,6 +1,6 @@
 {
   "db_name": "MySQL",
-  "query": "\nSELECT\n    users.token as user_token,\n    users.username,\n    users.display_name,\n\n    users.email_address,\n    users.email_confirmed,\n    users.email_gravatar_hash,\n\n    users.maybe_stripe_customer_id,\n    users.maybe_loyalty_program_key,\n\n    users.disable_gravatar,\n    users.auto_play_audio_preference,\n    users.auto_play_video_preference,\n    users.preferred_tts_result_visibility as `preferred_tts_result_visibility: enums::common::visibility::Visibility`,\n    users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: enums::common::visibility::Visibility`,\n\n    users.user_role_slug,\n    users.is_banned,\n\n    user_roles.can_use_tts,\n    user_roles.can_use_w2l,\n    user_roles.can_delete_own_tts_results,\n    user_roles.can_delete_own_w2l_results,\n    user_roles.can_delete_own_account,\n\n    user_roles.can_upload_tts_models,\n    user_roles.can_upload_w2l_templates,\n    user_roles.can_delete_own_tts_models,\n    user_roles.can_delete_own_w2l_templates,\n\n    user_roles.can_approve_w2l_templates,\n    user_roles.can_edit_other_users_profiles,\n    user_roles.can_edit_other_users_tts_models,\n    user_roles.can_edit_other_users_w2l_templates,\n    user_roles.can_delete_other_users_tts_models,\n    user_roles.can_delete_other_users_tts_results,\n    user_roles.can_delete_other_users_w2l_templates,\n    user_roles.can_delete_other_users_w2l_results,\n    user_roles.can_ban_users,\n    user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n    ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n    ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n    AND user_sessions.deleted_at IS NULL\n    AND users.user_deleted_at IS NULL\n    AND users.mod_deleted_at IS NULL\n        ",
+  "query": "\nSELECT\n    users.token as user_token,\n    users.username,\n    users.display_name,\n\n    users.email_address,\n    users.email_confirmed,\n    users.email_gravatar_hash,\n\n    users.maybe_stripe_customer_id,\n    users.maybe_loyalty_program_key,\n\n    users.disable_gravatar,\n    users.auto_play_audio_preference,\n    users.auto_play_video_preference,\n    users.preferred_tts_result_visibility as `preferred_tts_result_visibility: enums::common::visibility::Visibility`,\n    users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: enums::common::visibility::Visibility`,\n\n    users.user_role_slug,\n    users.is_banned,\n\n    users.can_access_studio,\n\n    user_roles.can_use_tts,\n    user_roles.can_use_w2l,\n    user_roles.can_delete_own_tts_results,\n    user_roles.can_delete_own_w2l_results,\n    user_roles.can_delete_own_account,\n\n    user_roles.can_upload_tts_models,\n    user_roles.can_upload_w2l_templates,\n    user_roles.can_delete_own_tts_models,\n    user_roles.can_delete_own_w2l_templates,\n\n    user_roles.can_approve_w2l_templates,\n    user_roles.can_edit_other_users_profiles,\n    user_roles.can_edit_other_users_tts_models,\n    user_roles.can_edit_other_users_w2l_templates,\n    user_roles.can_delete_other_users_tts_models,\n    user_roles.can_delete_other_users_tts_results,\n    user_roles.can_delete_other_users_w2l_templates,\n    user_roles.can_delete_other_users_w2l_results,\n    user_roles.can_ban_users,\n    user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n    ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n    ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n    AND user_sessions.deleted_at IS NULL\n    AND users.user_deleted_at IS NULL\n    AND users.mod_deleted_at IS NULL\n        ",
   "describe": {
     "columns": [
       {
@@ -155,6 +155,16 @@
       },
       {
         "ordinal": 15,
+        "name": "can_access_studio",
+        "type_info": {
+          "type": "Tiny",
+          "flags": "NOT_NULL | MULTIPLE_KEY",
+          "char_set": 63,
+          "max_size": 1
+        }
+      },
+      {
+        "ordinal": 16,
         "name": "can_use_tts",
         "type_info": {
           "type": "Tiny",
@@ -164,7 +174,7 @@
         }
       },
       {
-        "ordinal": 16,
+        "ordinal": 17,
         "name": "can_use_w2l",
         "type_info": {
           "type": "Tiny",
@@ -174,7 +184,7 @@
         }
       },
       {
-        "ordinal": 17,
+        "ordinal": 18,
         "name": "can_delete_own_tts_results",
         "type_info": {
           "type": "Tiny",
@@ -184,7 +194,7 @@
         }
       },
       {
-        "ordinal": 18,
+        "ordinal": 19,
         "name": "can_delete_own_w2l_results",
         "type_info": {
           "type": "Tiny",
@@ -194,7 +204,7 @@
         }
       },
       {
-        "ordinal": 19,
+        "ordinal": 20,
         "name": "can_delete_own_account",
         "type_info": {
           "type": "Tiny",
@@ -204,7 +214,7 @@
         }
       },
       {
-        "ordinal": 20,
+        "ordinal": 21,
         "name": "can_upload_tts_models",
         "type_info": {
           "type": "Tiny",
@@ -214,7 +224,7 @@
         }
       },
       {
-        "ordinal": 21,
+        "ordinal": 22,
         "name": "can_upload_w2l_templates",
         "type_info": {
           "type": "Tiny",
@@ -224,7 +234,7 @@
         }
       },
       {
-        "ordinal": 22,
+        "ordinal": 23,
         "name": "can_delete_own_tts_models",
         "type_info": {
           "type": "Tiny",
@@ -234,7 +244,7 @@
         }
       },
       {
-        "ordinal": 23,
+        "ordinal": 24,
         "name": "can_delete_own_w2l_templates",
         "type_info": {
           "type": "Tiny",
@@ -244,7 +254,7 @@
         }
       },
       {
-        "ordinal": 24,
+        "ordinal": 25,
         "name": "can_approve_w2l_templates",
         "type_info": {
           "type": "Tiny",
@@ -254,7 +264,7 @@
         }
       },
       {
-        "ordinal": 25,
+        "ordinal": 26,
         "name": "can_edit_other_users_profiles",
         "type_info": {
           "type": "Tiny",
@@ -264,7 +274,7 @@
         }
       },
       {
-        "ordinal": 26,
+        "ordinal": 27,
         "name": "can_edit_other_users_tts_models",
         "type_info": {
           "type": "Tiny",
@@ -274,7 +284,7 @@
         }
       },
       {
-        "ordinal": 27,
+        "ordinal": 28,
         "name": "can_edit_other_users_w2l_templates",
         "type_info": {
           "type": "Tiny",
@@ -284,7 +294,7 @@
         }
       },
       {
-        "ordinal": 28,
+        "ordinal": 29,
         "name": "can_delete_other_users_tts_models",
         "type_info": {
           "type": "Tiny",
@@ -294,7 +304,7 @@
         }
       },
       {
-        "ordinal": 29,
+        "ordinal": 30,
         "name": "can_delete_other_users_tts_results",
         "type_info": {
           "type": "Tiny",
@@ -304,7 +314,7 @@
         }
       },
       {
-        "ordinal": 30,
+        "ordinal": 31,
         "name": "can_delete_other_users_w2l_templates",
         "type_info": {
           "type": "Tiny",
@@ -314,7 +324,7 @@
         }
       },
       {
-        "ordinal": 31,
+        "ordinal": 32,
         "name": "can_delete_other_users_w2l_results",
         "type_info": {
           "type": "Tiny",
@@ -324,7 +334,7 @@
         }
       },
       {
-        "ordinal": 32,
+        "ordinal": 33,
         "name": "can_ban_users",
         "type_info": {
           "type": "Tiny",
@@ -334,7 +344,7 @@
         }
       },
       {
-        "ordinal": 33,
+        "ordinal": 34,
         "name": "can_delete_users",
         "type_info": {
           "type": "Tiny",
@@ -363,6 +373,7 @@
       false,
       false,
       false,
+      false,
       true,
       true,
       true,
@@ -384,5 +395,5 @@
       true
     ]
   },
-  "hash": "07d330b4c2e6b0dc76e2e61a3b44b47b8d287adf5eb0646f6cf2876655581a13"
+  "hash": "423109f2dcee23a558c43ecaab0eb2836714d5316620daaa1127b447cad59212"
 }

.sqlx/query-4673b503ee7c44661b1d496aafc07cda7be410af1d46f3eeceee6895f3a07037.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  inference_category = ?,\n  maybe_model_type = ?,\n  maybe_model_token = ?,\n\n  maybe_input_source_token = ?,\n  maybe_input_source_token_type = ?,\n\n  maybe_download_url = ?,\n\n  maybe_raw_inference_text = ?,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  priority_level = ?,\n  is_keepalive_required = ?,\n  max_duration_seconds = ?,\n\n  is_debug_request = ?,\n  maybe_routing_tag = ?,\n\n  status = \"pending\"\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 20
+    },
+    "nullable": []
+  },
+  "hash": "4673b503ee7c44661b1d496aafc07cda7be410af1d46f3eeceee6895f3a07037"
+}

.sqlx/query-567937a5918833330b74f94724eb69628aae49705ec4f8028321c38625eaa60c.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\n    INSERT INTO media_files\n    SET\n      token = ?,\n    \n      origin_category = ?,\n      origin_product_category = ?,\n      maybe_origin_model_type = ?,\n    \n      media_type = ?,\n      maybe_mime_type = ?,\n      file_size_bytes = ?,\n    \n      checksum_sha2 = ?,\n    \n      public_bucket_directory_hash = ?,\n      maybe_public_bucket_prefix = ?,\n      maybe_public_bucket_extension = ?,\n    \n      maybe_creator_user_token = ?,\n      maybe_creator_anonymous_visitor_token = ?,\n      creator_ip_address = ?,\n    \n      creator_set_visibility = ?,\n    \n      maybe_creator_file_synthetic_id = ?,\n      maybe_creator_category_synthetic_id = ?,\n    \n      is_generated_on_prem = ?,\n      generated_by_worker = ?,\n      generated_by_cluster = ?\n    \n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 20
-    },
-    "nullable": []
-  },
-  "hash": "567937a5918833330b74f94724eb69628aae49705ec4f8028321c38625eaa60c"
-}

.sqlx/query-5fcdce6dd4c2b24d9062fda7b0a080bda3f04da6933b4dba8d335f9b7aecacb3.json

@@ -1,6 +1,6 @@
 {
   "db_name": "MySQL",
-  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    jobs.attempt_count,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n\n    jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n    jobs.maybe_model_type,\n    jobs.maybe_model_token,\n    jobs.maybe_raw_inference_text,\n\n    jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n    jobs.on_success_result_entity_type as maybe_result_entity_type,\n    jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n    tts_models.title as maybe_tts_model_title,\n    voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n    tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n    voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n    media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n    media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n    media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n    jobs.assigned_worker as maybe_assigned_worker,\n    jobs.assigned_cluster as maybe_assigned_cluster,\n\n    jobs.is_keepalive_required,\n\n    jobs.created_at,\n    jobs.updated_at,\n\n    jobs.first_started_at as maybe_first_started_at,\n    jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n        ",
+  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    jobs.attempt_count,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n\n    jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n    jobs.maybe_model_type,\n    jobs.maybe_model_token,\n    jobs.maybe_raw_inference_text,\n\n    jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n    jobs.on_success_result_entity_type as maybe_result_entity_type,\n    jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n    model_weights.title as maybe_model_weights_title,\n    tts_models.title as maybe_tts_model_title,\n    voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n    tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n    voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n    media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n    media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n    media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n    jobs.assigned_worker as maybe_assigned_worker,\n    jobs.assigned_cluster as maybe_assigned_cluster,\n\n    jobs.is_keepalive_required,\n\n    jobs.created_at,\n    jobs.updated_at,\n\n    jobs.first_started_at as maybe_first_started_at,\n    jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n        ",
   "describe": {
     "columns": [
       {
@@ -135,7 +135,7 @@
       },
       {
         "ordinal": 13,
-        "name": "maybe_tts_model_title",
+        "name": "maybe_model_weights_title",
         "type_info": {
           "type": "VarString",
           "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -145,7 +145,7 @@
       },
       {
         "ordinal": 14,
-        "name": "maybe_voice_conversion_model_title",
+        "name": "maybe_tts_model_title",
         "type_info": {
           "type": "VarString",
           "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -155,7 +155,7 @@
       },
       {
         "ordinal": 15,
-        "name": "maybe_tts_public_bucket_path",
+        "name": "maybe_voice_conversion_model_title",
         "type_info": {
           "type": "VarString",
           "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -165,6 +165,16 @@
       },
       {
         "ordinal": 16,
+        "name": "maybe_tts_public_bucket_path",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 17,
         "name": "maybe_voice_conversion_public_bucket_hash",
         "type_info": {
           "type": "VarString",
@@ -174,7 +184,7 @@
         }
       },
       {
-        "ordinal": 17,
+        "ordinal": 18,
         "name": "maybe_media_file_public_bucket_directory_hash",
         "type_info": {
           "type": "VarString",
@@ -184,7 +194,7 @@
         }
       },
       {
-        "ordinal": 18,
+        "ordinal": 19,
         "name": "maybe_media_file_public_bucket_prefix",
         "type_info": {
           "type": "VarString",
@@ -194,7 +204,7 @@
         }
       },
       {
-        "ordinal": 19,
+        "ordinal": 20,
         "name": "maybe_media_file_public_bucket_extension",
         "type_info": {
           "type": "VarString",
@@ -204,7 +214,7 @@
         }
       },
       {
-        "ordinal": 20,
+        "ordinal": 21,
         "name": "maybe_assigned_worker",
         "type_info": {
           "type": "VarString",
@@ -214,7 +224,7 @@
         }
       },
       {
-        "ordinal": 21,
+        "ordinal": 22,
         "name": "maybe_assigned_cluster",
         "type_info": {
           "type": "VarString",
@@ -224,7 +234,7 @@
         }
       },
       {
-        "ordinal": 22,
+        "ordinal": 23,
         "name": "is_keepalive_required",
         "type_info": {
           "type": "Tiny",
@@ -234,7 +244,7 @@
         }
       },
       {
-        "ordinal": 23,
+        "ordinal": 24,
         "name": "created_at",
         "type_info": {
           "type": "Timestamp",
@@ -244,7 +254,7 @@
         }
       },
       {
-        "ordinal": 24,
+        "ordinal": 25,
         "name": "updated_at",
         "type_info": {
           "type": "Timestamp",
@@ -254,7 +264,7 @@
         }
       },
       {
-        "ordinal": 25,
+        "ordinal": 26,
         "name": "maybe_first_started_at",
         "type_info": {
           "type": "Timestamp",
@@ -264,7 +274,7 @@
         }
       },
       {
-        "ordinal": 26,
+        "ordinal": 27,
         "name": "maybe_successfully_completed_at",
         "type_info": {
           "type": "Timestamp",
@@ -300,6 +310,7 @@
       true,
       true,
       true,
+      true,
       false,
       false,
       false,
@@ -307,5 +318,5 @@
       true
     ]
   },
-  "hash": "4932e975b3369b06fcc23f7797f3a53a54211236383385b8740959b824bd9744"
+  "hash": "5fcdce6dd4c2b24d9062fda7b0a080bda3f04da6933b4dba8d335f9b7aecacb3"
 }

.sqlx/query-6c0a2cb31a3241fc6cd0b6738901a31bd91298f131d3b87744c698e92eaec34b.json

@@ -157,7 +157,7 @@
         "ordinal": 15,
         "name": "file_size_bytes",
         "type_info": {
-          "type": "Long",
+          "type": "LongLong",
           "flags": "NOT_NULL",
           "char_set": 63,
           "max_size": 10

.sqlx/query-8aad4a39b2e43743a2efc1b37c17860c15a2314145c61ab4a25e449a4cb22128.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO media_files\nSET\n  token = ?,\n\n  origin_category = ?,\n  origin_product_category = ?,\n\n  maybe_origin_model_type = ?,\n  maybe_origin_model_token = ?,\n\n  maybe_text_transcript = ?,\n\n  media_type = ?,\n  maybe_mime_type = ?,\n  file_size_bytes = ?,\n\n  checksum_sha2 = ?,\n\n  public_bucket_directory_hash = ?,\n  maybe_public_bucket_prefix = ?,\n  maybe_public_bucket_extension = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  maybe_creator_file_synthetic_id = ?,\n  maybe_creator_category_synthetic_id = ?,\n\n  is_generated_on_prem = ?,\n  generated_by_worker = ?,\n  generated_by_cluster = ?\n\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 22
+    },
+    "nullable": []
+  },
+  "hash": "8aad4a39b2e43743a2efc1b37c17860c15a2314145c61ab4a25e449a4cb22128"
+}

.sqlx/query-8bd7ddec0966c742754a278000902c7d5059237ef312c638817a465dcb1979de.json

@@ -157,7 +157,7 @@
         "ordinal": 15,
         "name": "file_size_bytes",
         "type_info": {
-          "type": "Long",
+          "type": "LongLong",
           "flags": "NOT_NULL",
           "char_set": 63,
           "max_size": 10

.sqlx/query-a3cf13dd4272af3b8fbcddba1f1ea711e64f57fb114886d60d4ae2937c56841d.json

@@ -157,7 +157,7 @@
         "ordinal": 15,
         "name": "file_size_bytes",
         "type_info": {
-          "type": "Long",
+          "type": "LongLong",
           "flags": "NOT_NULL",
           "char_set": 63,
           "max_size": 10

.sqlx/query-e32645b06d20a0037c3ca3d514b56af2dba8396f11a4c2a93ee4fab9b4579fc3.json

@@ -157,7 +157,7 @@
         "ordinal": 15,
         "name": "file_size_bytes",
         "type_info": {
-          "type": "Long",
+          "type": "LongLong",
           "flags": "NOT_NULL",
           "char_set": 63,
           "max_size": 10

.sqlx/query-e7f9bb387905704e9da3fd1657432a3d821d49462fcf82d8cca8dd2dcfb3df17.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\n    INSERT INTO media_files\n    SET\n      token = ?,\n    \n      origin_category = ?,\n      origin_product_category = ?,\n\n      maybe_origin_model_type = ?,\n\n      maybe_text_transcript = ?,\n\n      media_type = ?,\n      maybe_mime_type = ?,\n      file_size_bytes = ?,\n    \n      checksum_sha2 = ?,\n    \n      public_bucket_directory_hash = ?,\n      maybe_public_bucket_prefix = ?,\n      maybe_public_bucket_extension = ?,\n    \n      maybe_creator_user_token = ?,\n      maybe_creator_anonymous_visitor_token = ?,\n      creator_ip_address = ?,\n    \n      creator_set_visibility = ?,\n    \n      maybe_creator_file_synthetic_id = ?,\n      maybe_creator_category_synthetic_id = ?,\n    \n      is_generated_on_prem = ?,\n      generated_by_worker = ?,\n      generated_by_cluster = ?\n    \n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 21
+    },
+    "nullable": []
+  },
+  "hash": "e7f9bb387905704e9da3fd1657432a3d821d49462fcf82d8cca8dd2dcfb3df17"
+}

.sqlx/query-e97a0974ed643b8a20d4d319b4aa93aba979d5f8fc0bdc44b77ef4f945b03b35.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  inference_category = ?,\n  maybe_model_type = ?,\n  maybe_model_token = ?,\n\n  maybe_input_source_token = ?,\n  maybe_input_source_token_type = ?,\n\n  maybe_raw_inference_text = ?,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  priority_level = ?,\n  is_keepalive_required = ?,\n  max_duration_seconds = ?,\n\n  is_debug_request = ?,\n  maybe_routing_tag = ?,\n\n  status = \"pending\"\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 19
-    },
-    "nullable": []
-  },
-  "hash": "e97a0974ed643b8a20d4d319b4aa93aba979d5f8fc0bdc44b77ef4f945b03b35"
-}

Cargo.lock

@@ -762,6 +762,51 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "axum"
+version = "0.6.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bitflags 1.3.2",
+ "bytes 1.5.0",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "itoa 1.0.9",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
+dependencies = [
+ "async-trait",
+ "bytes 1.5.0",
+ "futures-util",
+ "http",
+ "http-body",
+ "mime",
+ "rustversion",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.69"
@@ -2694,6 +2739,12 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
 
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
 [[package]]
 name = "google_drive_common"
 version = "0.0.1"
@@ -2718,7 +2769,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 2.1.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -2972,6 +3023,18 @@ dependencies = [
  "want",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.5.0"
@@ -3067,6 +3130,16 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
 
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.1.0"
@@ -3140,6 +3213,9 @@ dependencies = [
  "my-workspace-hack",
  "mysql_queries",
  "once_cell",
+ "opentelemetry",
+ "opentelemetry-otlp",
+ "opentelemetry_sdk",
  "r2d2_redis",
  "rand 0.8.5",
  "redis_common",
@@ -3567,6 +3643,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
 
+[[package]]
+name = "matchit"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
+
 [[package]]
 name = "maybe-async"
 version = "0.2.7"
@@ -3878,7 +3960,7 @@ dependencies = [
  "heck",
  "hmac 0.12.1",
  "hyper",
- "indexmap",
+ "indexmap 2.1.0",
  "lazy_static",
  "libc",
  "log",
@@ -4241,6 +4323,93 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "opentelemetry"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "indexmap 2.1.0",
+ "js-sys",
+ "once_cell",
+ "pin-project-lite",
+ "thiserror",
+ "urlencoding",
+]
+
+[[package]]
+name = "opentelemetry-otlp"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930"
+dependencies = [
+ "async-trait",
+ "futures-core",
+ "http",
+ "opentelemetry",
+ "opentelemetry-proto",
+ "opentelemetry-semantic-conventions",
+ "opentelemetry_sdk",
+ "prost",
+ "thiserror",
+ "tokio",
+ "tonic",
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1"
+dependencies = [
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "prost",
+ "tonic",
+]
+
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84"
+dependencies = [
+ "opentelemetry",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4"
+dependencies = [
+ "async-trait",
+ "crossbeam-channel",
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "glob",
+ "once_cell",
+ "opentelemetry",
+ "ordered-float",
+ "percent-encoding",
+ "rand 0.8.5",
+ "thiserror",
+ "tokio",
+ "tokio-stream",
+]
+
+[[package]]
+name = "ordered-float"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "ordered-multimap"
 version = "0.4.3"
@@ -4598,6 +4767,29 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
+dependencies = [
+ "bytes 1.5.0",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
+dependencies = [
+ "anyhow",
+ "itertools 0.10.5",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "psm"
 version = "0.1.21"
@@ -5448,7 +5640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "224e6a14f315852940f3ec103125aa6482f0e224732ed91ed3330ed633077c34"
 dependencies = [
  "form_urlencoded",
- "indexmap",
+ "indexmap 2.1.0",
  "itoa 1.0.9",
  "ryu",
  "serde",
@@ -5460,7 +5652,7 @@ version = "1.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
  "itoa 1.0.9",
  "ryu",
  "serde",
@@ -5886,7 +6078,7 @@ dependencies = [
  "futures-util",
  "hashlink",
  "hex",
- "indexmap",
+ "indexmap 2.1.0",
  "log",
  "memchr",
  "native-tls",
@@ -6485,6 +6677,12 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
 [[package]]
 name = "system-configuration"
 version = "0.5.1"
@@ -6728,6 +6926,16 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-macros"
 version = "2.2.0"
@@ -6899,13 +7107,67 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
  "winnow",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
+dependencies = [
+ "async-trait",
+ "axum",
+ "base64 0.21.5",
+ "bytes 1.5.0",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap 1.9.3",
+ "pin-project",
+ "pin-project-lite",
+ "rand 0.8.5",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+
 [[package]]
 name = "tower-service"
 version = "0.3.2"
@@ -7342,6 +7604,8 @@ dependencies = [
  "toml 0.5.11",
  "user_input_common",
  "user_traits_component",
+ "utoipa",
+ "utoipa-swagger-ui",
 ]
 
 [[package]]
@@ -7362,7 +7626,7 @@ version = "4.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
  "serde",
  "serde_json",
  "utoipa-gen",

_elasticsearch/index_definitions/model_weights_v1.json

@@ -34,7 +34,13 @@
       "created_at" : { "type" : "date" },
       "updated_at" : { "type" : "date" },
       "user_deleted_at" : { "type" : "date" },
-      "mod_deleted_at" : { "type" : "date" }
+      "mod_deleted_at" : { "type" : "date" },
+
+      "is_deleted" : {
+        "type": "boolean",
+        "index": true
+      }
+
     }
   }
 }

_sql/actions/studio/enroll_users_in_studio.sql

@@ -0,0 +1,28 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Staff
+-- teddanson is victor's alt
+-- SELECT token, username FROM users WHERE username IN (
+UPDATE users SET can_access_studio = true WHERE username IN (
+    'bflat',
+    'brandon',
+    'crossproduct',
+    'crossproduct1',
+    'echelon',
+    'fyscott',
+    'olivicmic',
+    'saltacc',
+    'teddanson',
+    'vegito1089',
+    'wilwong',
+    'yae_ph',
+    'zzz_last_item'
+);
+
+UPDATE users SET can_access_studio = true WHERE username IN (
+    'endtimes',
+    'tropicalfun',
+    'zzz_last_item'
+);

_sql/migrations/2024-02-06-002543_alter_users_add_rollout_flag/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users DROP COLUMN can_access_studio;

_sql/migrations/2024-02-06-002543_alter_users_add_rollout_flag/up.sql

@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users
+    ADD COLUMN can_access_studio BOOLEAN NOT NULL DEFAULT false
+    AFTER user_role_slug;
+
+ALTER TABLE users
+    ADD INDEX index_can_access_studio (can_access_studio);

_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/down.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    MODIFY COLUMN file_size_bytes INT(10) NOT NULL DEFAULT 0;

_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/up.sql

@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    MODIFY COLUMN file_size_bytes BIGINT(10) NOT NULL DEFAULT 0;

_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN maybe_text_transcript;

_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/up.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN maybe_text_transcript TEXT DEFAULT NULL
+    AFTER maybe_frame_height;

_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN maybe_batch_token;

_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/up.sql

@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN maybe_batch_token VARCHAR(32) DEFAULT NULL
+    AFTER is_batch_generated;
+
+ALTER TABLE media_files
+    ADD INDEX index_maybe_batch_token (maybe_batch_token);

_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN maybe_download_url;

_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/up.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN maybe_download_url VARCHAR(1024) DEFAULT NULL
+    AFTER maybe_raw_inference_text;

_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN on_success_result_batch_token;

_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/up.sql

@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN on_success_result_batch_token VARCHAR(32) DEFAULT NULL
+    AFTER on_success_result_entity_token;

_sql/migrations_squashed/generic_inference_jobs.sql

@@ -72,6 +72,11 @@ CREATE TABLE generic_inference_jobs (
   -- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
   maybe_raw_inference_text TEXT DEFAULT NULL,
 
+  -- For download-related jobs, this is the URL
+  -- This is nullable because not all inference jobs have a download URL.
+  -- The length of 1024 is arbitrary, but we shouldn't need anything longer.
+  maybe_download_url VARCHAR(1024) DEFAULT NULL,
+
   -- ========== SUCCESS CASE ==========
 
   -- The type of the object will vary based on the type of the upload,
@@ -82,6 +87,11 @@ CREATE TABLE generic_inference_jobs (
   -- This is only populated on a successful result.
   on_success_result_entity_token VARCHAR(32) DEFAULT NULL,
 
+  -- If the job completes successfully, this is the batch token (ie. when
+  -- several results are created).
+  -- Only populated if there are actual batches generated.
+  on_success_result_batch_token VARCHAR(32) DEFAULT NULL,
+
   -- ========== CREATOR DETAILS AND PREFERENCES ==========
 
   -- Foreign key to user

_sql/migrations_squashed/media_files.sql

@@ -58,11 +58,15 @@ CREATE TABLE media_files (
   -- The original filename of the media (if uploaded by a user)
   maybe_origin_filename VARCHAR(255) DEFAULT NULL,
 
+  -- TODO: Remove after `maybe_batch_token` gains use.
   -- Whether this media file was generated as part of a batch
   -- If so, we can look up the batch in a separate query/call to the `batch_generations` table.
   -- We won't hold the batch token in this table since it'll be very sparsely populated.
   is_batch_generated BOOLEAN NOT NULL DEFAULT FALSE,
 
+  -- If the media file is generated as part of a batch, this designates the batch.
+  maybe_batch_token VARCHAR(32) DEFAULT NULL,
+
   -- ========== METADATA ==========
 
   -- We can store text transcripts, etc. in other tables and refer back via foreign key.
@@ -100,6 +104,9 @@ CREATE TABLE media_files (
   -- For videos, the original frame height.
   maybe_frame_height INT(5) DEFAULT NULL,
 
+  -- Optional text transcript for audio or video (especially TTS)
+  maybe_text_transcript TEXT DEFAULT NULL,
+
   -- Checksum of the original media
   -- SHA1 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
   -- checksum_sha1 CHAR(40) NOT NULL,
@@ -212,6 +219,7 @@ CREATE TABLE media_files (
   KEY index_maybe_origin_model_type (maybe_origin_model_type),
   KEY fk_maybe_origin_model_token (maybe_origin_model_token),
   KEY fk_maybe_origin_model_type_and_token (maybe_origin_model_type, maybe_origin_model_token),
+  KEY index_maybe_batch_token (maybe_batch_token),
   KEY index_media_type (media_type),
   KEY index_checksum_sha2 (checksum_sha2),
   KEY fk_maybe_creator_user_token (maybe_creator_user_token),

_sql/migrations_squashed/model_weights/model_weights.sql

@@ -95,9 +95,8 @@ CREATE TABLE model_weights (
   -- The filename that was used at upload time (if available)
   original_filename VARCHAR(255) DEFAULT NULL,
 
-  -- TODO: Make this a BIGINT so we can store more than 2GB.
-  -- File characteristics
-  file_size_bytes INT(10) NOT NULL DEFAULT 0,
+  -- File size. BIGINT to store more than 2GB size.
+  file_size_bytes BIGINT(10) NOT NULL DEFAULT 0,
 
   -- Checksum of the original weights file
   -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]

_sql/migrations_squashed/user_tables/users.sql

@@ -22,6 +22,10 @@ CREATE TABLE users (
   -- The role assigned to the user confers permissions.
   user_role_slug VARCHAR(16) NOT NULL,
 
+  -- Rollout flag for Storyteller Studio
+  -- Default to false until rollout is 100%, then remove the column.
+  can_access_studio BOOLEAN NOT NULL DEFAULT false,
+
   -- ========== PREMIUM FEATURES ==========
 
   -- If the user has a Stripe subscription (or has had one), we link it to the user here.
@@ -171,6 +175,7 @@ CREATE TABLE users (
   UNIQUE KEY (username),
   UNIQUE KEY (email_address),
   KEY fk_user_role_slug (user_role_slug),
+  KEY index_can_access_studio (can_access_studio),
   KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token),
   KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token)
 

crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs

@@ -40,6 +40,9 @@ pub enum InferenceCategory {
 
   #[serde(rename = "format_conversion")]
   FormatConversion,
+
+  #[serde(rename = "convert_bvh_to_workflow")]
+  ConvertBvhToWorkflow,
 }
 
 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -58,6 +61,7 @@ impl InferenceCategory {
       Self::Mocap => "mocap",
       Self::Workflow => "workflow",
       Self::FormatConversion => "format_conversion",
+      Self::ConvertBvhToWorkflow => "convert_bvh_to_workflow",
     }
   }
 
@@ -71,6 +75,7 @@ impl InferenceCategory {
       "mocap" => Ok(Self::Mocap),
       "workflow" => Ok(Self::Workflow),
       "format_conversion" => Ok(Self::FormatConversion),
+      "convert_bvh_to_workflow" => Ok(Self::ConvertBvhToWorkflow),
       _ => Err(format!("invalid value: {:?}", value)),
     }
   }
@@ -87,6 +92,7 @@ impl InferenceCategory {
       Self::Mocap,
       Self::Workflow,
       Self::FormatConversion,
+      Self::ConvertBvhToWorkflow,
     ])
   }
 }
@@ -109,6 +115,7 @@ mod tests {
       assert_serialization(InferenceCategory::Mocap, "mocap");
       assert_serialization(InferenceCategory::Workflow, "workflow");
       assert_serialization(InferenceCategory::FormatConversion, "format_conversion");
+      assert_serialization(InferenceCategory::ConvertBvhToWorkflow, "convert_bvh_to_workflow");
     }
 
     #[test]
@@ -121,6 +128,7 @@ mod tests {
       assert_eq!(InferenceCategory::Mocap.to_str(), "mocap");
       assert_eq!(InferenceCategory::Workflow.to_str(), "workflow");
       assert_eq!(InferenceCategory::FormatConversion.to_str(), "format_conversion");
+      assert_eq!(InferenceCategory::ConvertBvhToWorkflow.to_str(), "convert_bvh_to_workflow");
     }
 
     #[test]
@@ -133,6 +141,7 @@ mod tests {
       assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap);
       assert_eq!(InferenceCategory::from_str("workflow").unwrap(), InferenceCategory::Workflow);
       assert_eq!(InferenceCategory::from_str("format_conversion").unwrap(), InferenceCategory::FormatConversion);
+      assert_eq!(InferenceCategory::from_str("convert_bvh_to_workflow").unwrap(), InferenceCategory::ConvertBvhToWorkflow);
     }
 
     #[test]
@@ -148,6 +157,7 @@ mod tests {
       assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::Workflow));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::FormatConversion));
+      assert_eq!(variants.pop_first(), Some(InferenceCategory::ConvertBvhToWorkflow));
       assert_eq!(variants.pop_first(), None);
 
       // Generated check

crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_type.rs

@@ -15,6 +15,10 @@ use strum::EnumIter;
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)]
 #[serde(rename_all = "snake_case")]
 pub enum InferenceJobType {
+  /// Job that converts bevy to workflow files
+  #[serde(rename = "bevy_to_workflow")]
+  BevyToWorkflow,
+
   /// Jobs that run ComfyUI workflows
   ComfyUi,
 
@@ -64,6 +68,7 @@ impl_mysql_enum_coders!(InferenceJobType);
 impl InferenceJobType {
   pub fn to_str(&self) -> &'static str {
     match self {
+      Self::BevyToWorkflow => "bevy_to_workflow",
       Self::ComfyUi => "comfy_ui",
       Self::ConvertFbxToGltf => "convert_fbx_gltf",
       Self::MocapNet => "mocap_net",
@@ -80,6 +85,7 @@ impl InferenceJobType {
 
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
+      "bevy_to_workflow" => Ok(Self::BevyToWorkflow),
       "comfy_ui" => Ok(Self::ComfyUi),
       "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
       "mocap_net" => Ok(Self::MocapNet),
@@ -99,6 +105,7 @@ impl InferenceJobType {
     // NB: BTreeSet is sorted
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
+      Self::BevyToWorkflow,
       Self::ComfyUi,
       Self::ConvertFbxToGltf,
       Self::MocapNet,
@@ -129,6 +136,7 @@ mod tests {
 
     #[test]
     fn test_serialization() {
+      assert_serialization(InferenceJobType::BevyToWorkflow, "bevy_to_workflow");
       assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
       assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
       assert_serialization(InferenceJobType::MocapNet, "mocap_net");
@@ -144,6 +152,7 @@ mod tests {
 
     #[test]
     fn to_str() {
+      assert_eq!(InferenceJobType::BevyToWorkflow.to_str(), "bevy_to_workflow");
       assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
       assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
       assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
@@ -159,6 +168,7 @@ mod tests {
 
     #[test]
     fn from_str() {
+      assert_eq!(InferenceJobType::from_str("bevy_to_workflow").unwrap(), InferenceJobType::BevyToWorkflow);
       assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
       assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
       assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
@@ -176,7 +186,8 @@ mod tests {
     fn all_variants() {
       // Static check
       let mut variants = InferenceJobType::all_variants();
-      assert_eq!(variants.len(), 11);
+      assert_eq!(variants.len(), 12);
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::BevyToWorkflow));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));

crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs

@@ -44,6 +44,8 @@ pub enum InferenceModelType {
   /// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
   #[serde(rename = "convert_fbx_gltf")]
   ConvertFbxToGltf,
+  #[serde(rename = "bvh_to_workflow")]
+  BvhToWorkflow
 }
 
 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -66,6 +68,7 @@ impl InferenceModelType {
       Self::StyleTTS2 => "styletts2",
       Self::ComfyUi => "comfy_ui",
       Self::ConvertFbxToGltf => "convert_fbx_gltf",
+      Self::BvhToWorkflow => "bvh_to_workflow",
     }
   }
 
@@ -83,6 +86,7 @@ impl InferenceModelType {
       "styletts2" => Ok(Self::StyleTTS2),
       "comfy_ui" => Ok(Self::ComfyUi),
       "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
+      "bvh_to_workflow" => Ok(Self::BvhToWorkflow),
       _ => Err(format!("invalid value: {:?}", value)),
     }
   }
@@ -103,6 +107,7 @@ impl InferenceModelType {
       Self::StyleTTS2,
       Self::ComfyUi,
       Self::ConvertFbxToGltf,
+        Self::BvhToWorkflow,
     ])
   }
 }
@@ -129,6 +134,7 @@ mod tests {
       assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
       assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
       assert_serialization(InferenceModelType::ConvertFbxToGltf, "convert_fbx_gltf");
+      assert_serialization(InferenceModelType::BvhToWorkflow, "bvh_to_workflow");
     }
 
     #[test]
@@ -145,6 +151,7 @@ mod tests {
       assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
       assert_eq!(InferenceModelType::ComfyUi.to_str(), "comfy_ui");
       assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
+      assert_eq!(InferenceModelType::BvhToWorkflow.to_str(), "bvh_to_workflow");
     }
 
     #[test]
@@ -161,6 +168,7 @@ mod tests {
       assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
       assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
       assert_eq!(InferenceModelType::from_str("convert_fbx_gltf").unwrap(), InferenceModelType::ConvertFbxToGltf);
+      assert_eq!(InferenceModelType::from_str("bvh_to_workflow").unwrap(), InferenceModelType::BvhToWorkflow);
     }
 
     #[test]
@@ -180,6 +188,7 @@ mod tests {
       assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
       assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
+      assert_eq!(variants.pop_first(), Some(InferenceModelType::BvhToWorkflow));
       assert_eq!(variants.pop_first(), None);
 
       // Generated check

crates/_types/tokens/src/tokens/zs_voice_datasets.rs

@@ -2,11 +2,12 @@ use std::fmt::Debug;
 
 use serde::Deserialize;
 use serde::Serialize;
+use utoipa::ToSchema;
 
 use crate::prefixes::TokenPrefix;
 
 /// The primary key for the  "zs_voice_datasets" table.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
 #[sqlx(transparent)]
 pub struct ZsVoiceDatasetToken(pub String);
 

crates/cli/dev-database-seed/src/main.rs

@@ -7,8 +7,8 @@ use errors::AnyhowResult;
 
 use crate::bucket_clients::get_bucket_clients;
 use crate::cli_args::parse_cli_args;
-use crate::seeding::seed_model_weights_for_local::seed_weights;
-
+use crate::seeding::model_weights::seed_weights;
+use crate::seeding::users::seed_user_accounts;
 pub mod bucket_clients;
 
 pub mod cli_args;
@@ -47,11 +47,11 @@ pub async fn main() -> AnyhowResult<()> {
 //    maybe_elasticsearch = Some(get_elasticsearch_client());
 //  }
 
-  // seed_user_accounts(&pool).await?;
+  //seed_user_accounts(&pool).await?;
   // seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?;
   //seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?;
   // seed_voice_conversion(&pool).await?;
-  seed_weights(&pool).await?;
+  //seed_weights(&pool).await?;
   // seed_media_seedtool(&pool).await?;
   // seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?;
   

crates/cli/dev-database-seed/src/seeding/mod.rs

@@ -1,4 +1,4 @@
-pub mod seed_model_weights_for_local;
+pub mod model_weights;
 pub mod seed_media_files;
 pub mod tts_tacotron2;
 pub mod users;

crates/cli/dev-database-seed/src/seeding/model_weights.rs

@@ -41,14 +41,14 @@ pub async fn test_seed_weights_files() -> AnyhowResult<()> {
 
     let bucket_details1 = RemoteCloudBucketDetails {
         object_hash: String::from("apa0ej6es8d3ss2gwtf1cghge35qn9tn"),
-        prefix: String::from("sd15"),
-        suffix: String::from("safetensors"),
+        prefix: String::from("sd15_"),
+        suffix: String::from(".safetensors"),
     };
 
     let bucket_details2 = RemoteCloudBucketDetails {
         object_hash: String::from("27kz11et18fargyyxbj66ntfn621k9d3"),
-        prefix: String::from("loRA"),
-        suffix: String::from("safetensors"),
+        prefix: String::from("loRA_"),
+        suffix: String::from(".safetensors"),
     };
 
     remote_cloud_file_client.download_file(bucket_details1, String::from("./checkpoint")).await?;
@@ -126,7 +126,7 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
 
     // create a loop that loops from 1 to 100
     for i in 1..=100 {
-
+        // create a new weight
         
         let weights_category = match i {
             1..=20 => WeightsCategory::VoiceConversion,
@@ -156,6 +156,11 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
         let _private_bucket_hash = "".to_string();
         let private_bucket_prefix;
         let private_bucket_extension;
+
+        //let mut cached_user_ratings_total_count;
+        //let mut cached_user_ratings_positive_count;
+        //let mut cached_user_ratings_negative_count;
+        //let mut cached_user_ratings_ratio;
         let version;
 
         match i {
@@ -174,8 +179,14 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 
                 original_filename = format!("gura_gwar{}.safetensors", i);
                 original_download_url = format!("www.google.ca");
+
+                // private_bucket_hash = format!("bucket_hash{}", i);
                 private_bucket_prefix = format!("_fake");
                 private_bucket_extension = format!("rvcV2");
+                //cached_user_ratings_total_count = i;
+                //cached_user_ratings_positive_count = i;
+                //cached_user_ratings_negative_count = i;
+                //cached_user_ratings_ratio = i as u32 / 100;
                 version = i as i32;
                 println!("Seeding RVCv2 model {}", i);
             },
@@ -193,10 +204,18 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 "#;
 
                 description_rendered_html = tts_markdown_description;
+                
                 original_filename = format!("filename{}.txt", i);
                 original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16");
+
+                // private_bucket_hash = format!("bucket_hash{}", i);
                 private_bucket_prefix = format!("_fake");
                 private_bucket_extension = format!("tt2");
+
+                //cached_user_ratings_total_count = i;
+                //cached_user_ratings_positive_count = i;
+                //cached_user_ratings_negative_count = i;
+                //cached_user_ratings_ratio = i as u32 / 100;
                 version = i as i32;
                 println!("Seeding TT2 model {}", i);
             },
@@ -204,12 +223,22 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 // SD 1.5
                 model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                 title = format!("dragonfruitGTv1: {}", i);
+                
                 description = format!("Description Number {}:", i);
+                
                 description_rendered_html = sd1_5_markdown_description;
+                
                 original_filename = format!("filename{}.txt", i);
                 original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16");
+
+                // private_bucket_hash = format!("bucket_hash{}", i);
                 private_bucket_prefix = format!("_fake");
                 private_bucket_extension = format!("sd15");
+
+                //cached_user_ratings_total_count = i;
+                //cached_user_ratings_positive_count = i;
+                //cached_user_ratings_negative_count = i;
+                //cached_user_ratings_ratio = i as u32 / 100;
                 version = i as i32;
                 println!("Seeding SD15 model {}", i);
             },
@@ -233,14 +262,24 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 Have fun! 😊
                 "#;
                 let _sdxl_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg";
+
                 model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                 title = format!("SDXL_Niji_Special Edition: {}", i);
                 description = format!("Description Number {}:", i);
+
+                // private_bucket_hash = format!("bucket_hash{}", i);
                 private_bucket_prefix = format!("_fake");
                 private_bucket_extension = format!("sdxl");
+  
                 description_rendered_html = sdxl_markdown_description;
+                  
                 original_filename = format!("filename{}.txt", i);
                 original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16");
+  
+                //cached_user_ratings_total_count = i;
+                //cached_user_ratings_positive_count = i;
+                //cached_user_ratings_negative_count = i;
+                //cached_user_ratings_ratio = i as u32 / 100;
                 version = i as i32;
                 println!("Seeding SDXL model {}", i);
             },
@@ -250,15 +289,26 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.
                 - LoRA Gawr Gura
                 "#;
+
                 let _lora_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/123645df-dee2-4239-863a-76a150b09c32/width=1024/00000-2171948503.jpeg";
                 model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                 title = format!("Gawr Gura LoRA: {}", i);
+                
                 description = format!("Description Number {}:", i);
+
                 description_rendered_html = lora_markdown_description;
+                
                 original_filename = format!("filename{}.txt", i);
                 original_download_url = format!("https://civitai.com/api/download/models/149193?type=Model&format=SafeTensor&size=pruned&fp=fp16");
+
+                // private_bucket_hash = format!("bucket_hash{}", i);
                 private_bucket_prefix = format!("_fake");
                 private_bucket_extension = format!("loRA");
+
+                //cached_user_ratings_total_count = i;
+                //cached_user_ratings_positive_count = i;
+                //cached_user_ratings_negative_count = i;
+                //cached_user_ratings_ratio = i as u32 / 100;
                 version = i as i32;
                 println!("Seeding LoRA model {}", i);
             },
@@ -269,12 +319,16 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 - **Improved Quality:** The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.
                 - Gawr Gura MOS 4
                 "#;
+
                 model_weight_token = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
                 title = format!("Gawr Gura: {}", i);
                 description = format!("Description Number {} The special edition offers significantly better quality with fewer issues. It's the model as initially envisioned.", i);
                 description_rendered_html = rvc_markdown_description;
+                
                 original_filename = format!("gura_gwar{}.safetensors", i);
                 original_download_url = format!("www.google.ca");
+
+                // private_bucket_hash = format!("bucket_hash{}", i);
                 private_bucket_prefix = format!("_fake");
                 private_bucket_extension = format!("rvcV2");
                 version = i as i32;
@@ -301,6 +355,10 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                 public_bucket_hash: "bucket_hash1".to_string(),
                 maybe_public_bucket_prefix: Some(private_bucket_prefix),
                 maybe_public_bucket_extension: Some(private_bucket_extension),
+                //cached_user_ratings_total_count,
+                //cached_user_ratings_positive_count,
+                //cached_user_ratings_negative_count,
+                //maybe_cached_user_ratings_ratio: Some(cached_user_ratings_ratio as f32),
                 version: version as u32,
                 mysql_pool: &mysql_pool, // replace with actual MySqlPool
             };
@@ -528,9 +586,6 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
     let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
     let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
 
-    let model_weight_token4 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
-    let model_weight_token5 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
-    let model_weight_token6 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
 
     let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
 
@@ -538,11 +593,6 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
     let lora_descriptor = Box::new(WeightsLoRADescriptor{});
     let sd_vae_15_weights_descriptor = Box::new(WeightsSD15Descriptor {});
 
-
-    let sd_15_weights_descriptor2 = Box::new(WeightsSD15Descriptor {});
-    let lora_descriptor2 = Box::new(WeightsLoRADescriptor{});
-    let sd_vae_15_weights_descriptor2 = Box::new(WeightsSD15Descriptor {});
-
     let mut path_object_SD = get_seed_tool_data_root();
     path_object_SD.push("models/imagegen/sd15/majicmixFantasy_v30Vae.safetensors");
 
@@ -552,26 +602,10 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
     let mut path_to_VAE = get_seed_tool_data_root();
     path_to_VAE.push("models/imagegen/vae/zVae_v20.safetensors");
 
-    // anime model 
-    let mut path_object_SD_anime = get_seed_tool_data_root();
-    path_object_SD_anime.push("models/imagegen/sd15/animerge_v26.safetensors");
-
-    // hongkai clara
-    let mut path_object_loRA2 = get_seed_tool_data_root();
-    path_object_loRA2.push("models/imagegen/loRA/Clara_Honkai_Star_Rail_v2-10.safetensors");
-
-    // anime vae
-    let mut path_to_VAE2 =  get_seed_tool_data_root();
-    path_to_VAE2.push("models/imagegen/vae/vaeFtMse840000EmaPruned_vae.safetensors");
-
     let metadata1 = remote_cloud_file_client.upload_file(sd_15_weights_descriptor,path_object_SD.as_path().to_str().unwrap()).await?;
     let metadata2 = remote_cloud_file_client.upload_file(lora_descriptor,path_object_loRA.as_path().to_str().unwrap()).await?;
     let metadata3 = remote_cloud_file_client.upload_file(sd_vae_15_weights_descriptor,path_to_VAE.as_path().to_str().unwrap()).await?;
 
-    let metadata4 = remote_cloud_file_client.upload_file(sd_15_weights_descriptor2,path_object_SD_anime.as_path().to_str().unwrap()).await?;
-    let metadata5 = remote_cloud_file_client.upload_file(lora_descriptor2,path_object_loRA2.as_path().to_str().unwrap()).await?;
-    let metadata6 = remote_cloud_file_client.upload_file(sd_vae_15_weights_descriptor2,path_to_VAE2.as_path().to_str().unwrap()).await?;
-
     let weights1 = CreateModelWeightsArgs {
         token: &model_weight_token1, // replace with actual ModelWeightToken
         weights_type: WeightsType::StableDiffusion15, // replace with actual WeightsType
@@ -585,7 +619,7 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
         maybe_last_update_user_token: Some("Last Update User Token".to_string()),
         original_download_url: Some("https://civitai.com/models/41865/majicmix-fantasy".to_string()),
         original_filename: Some("majicmixFantasy_v30Vae.safetensors".to_string()),
-        file_size_bytes: 0,
+        file_size_bytes: metadata1.file_size_bytes,
         file_checksum_sha2: metadata1.sha256_checksum.to_string(),
         public_bucket_hash: metadata1.bucket_details.clone().unwrap().object_hash,
         maybe_public_bucket_prefix: Some(metadata1.bucket_details.clone().unwrap().prefix),
@@ -607,7 +641,7 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
         maybe_last_update_user_token: Some("<p> Honkai <p>".to_string()),
         original_download_url: Some("https://civitai.com/models/95243/mihoyo-collection-honkai-impact-3rd-or-honkai-star-rail-or-genshin-impact-or-zenless-zone-zero".to_string()),
         original_filename: Some("xiawolei-v100-000019.safetensors".to_string()),
-        file_size_bytes: 0,
+        file_size_bytes: metadata2.file_size_bytes,
         file_checksum_sha2: metadata2.sha256_checksum.to_string(),
         public_bucket_hash: metadata2.bucket_details.clone().unwrap().object_hash.clone(),
         maybe_public_bucket_prefix: Some(metadata2.bucket_details.clone().unwrap().prefix),
@@ -629,7 +663,7 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
         maybe_last_update_user_token: Some("Last Update User Token".to_string()),
         original_download_url: Some("https://civitai.com/models/97653/z-vae".to_string()),
         original_filename: Some("zVae_v20.safetensors".to_string()),
-        file_size_bytes: 0,
+        file_size_bytes: metadata3.file_size_bytes,
         file_checksum_sha2: metadata3.sha256_checksum.to_string(),
         public_bucket_hash: metadata3.bucket_details.clone().unwrap().object_hash,
         maybe_public_bucket_prefix: Some(metadata3.bucket_details.clone().unwrap().prefix),
@@ -638,88 +672,18 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
         mysql_pool: &mysql_pool, // replace with actual MySqlPool
     };
 
-
-    let weights4 = CreateModelWeightsArgs {
-        token: &model_weight_token4, // replace with actual ModelWeightToken
-        weights_type: WeightsType::StableDiffusion15, // replace with actual WeightsType
-        weights_category: WeightsCategory::ImageGeneration, // replace with actual WeightsCategory
-        title: "animerge_v26".to_string(),
-        maybe_description_markdown: Some(sd1_5_markdown_description.to_string()),
-        maybe_description_rendered_html: Some("<p>Description</p>".to_string()),
-        creator_user_token: Some(&user_token), // replace with actual UserToken
-        creator_ip_address: "192.168.1.1",
-        creator_set_visibility: Visibility::Public,
-        maybe_last_update_user_token: Some("Last Update User Token".to_string()),
-        original_download_url: Some("https://civitai.com/models/144249/animerge".to_string()),
-        original_filename: Some("".to_string()),
-        file_size_bytes: 0,
-        file_checksum_sha2: metadata4.sha256_checksum.to_string(),
-        public_bucket_hash: metadata4.bucket_details.clone().unwrap().object_hash,
-        maybe_public_bucket_prefix: Some(metadata4.bucket_details.clone().unwrap().prefix),
-        maybe_public_bucket_extension: Some(metadata4.bucket_details.clone().unwrap().suffix),
-        version: 1,
-        mysql_pool: &mysql_pool, // replace with actual MySqlPool
-    };
-
-    let weights5  = CreateModelWeightsArgs {
-        token: &model_weight_token5, // replace with actual ModelWeightToken
-        weights_type: WeightsType::LoRA, // replace with actual WeightsType
-        weights_category: WeightsCategory::ImageGeneration, // replace with actual WeightsCategory
-        title: "clara LoRA".to_string(),
-        maybe_description_markdown: Some(miyhoyo_description.to_string()),
-        maybe_description_rendered_html: Some(miyhoyo_description.to_string()),
-        creator_user_token: Some(&user_token), // replace with actual UserToken
-        creator_ip_address: "292.268.2.2",
-        creator_set_visibility: Visibility::Public,
-        maybe_last_update_user_token: Some("<p> Honkai <p>".to_string()),
-        original_download_url: Some("https://civitai.com/models/56454/clara-honkai-star-rail-lora".to_string()),
-        original_filename: Some("clara.safetensors".to_string()),
-        file_size_bytes: 0,
-        file_checksum_sha2: metadata5.sha256_checksum.to_string(),
-        public_bucket_hash: metadata5.bucket_details.clone().unwrap().object_hash.clone(),
-        maybe_public_bucket_prefix: Some(metadata5.bucket_details.clone().unwrap().prefix),
-        maybe_public_bucket_extension: Some(metadata5.bucket_details.clone().unwrap().suffix),
-        version: 2,
-        mysql_pool: &mysql_pool, // replace with actual MySqlPool
-    };
-
-    // some VAE
-    let weights6 = CreateModelWeightsArgs {
-        token: &model_weight_token6, // replace with actual ModelWeightToken
-        weights_type: WeightsType::StableDiffusion15, // replace with actual WeightsType
-        weights_category: WeightsCategory::ImageGeneration, // replace with actual WeightsCategory
-        title: "anime-vae".to_string(),
-        maybe_description_markdown: Some(sd_vae_description.to_string()),
-        maybe_description_rendered_html: Some("<p>Description</p>".to_string()),
-        creator_user_token: Some(&user_token), // replace with actual UserToken
-        creator_ip_address: "192.168.1.1",
-        creator_set_visibility: Visibility::Public,
-        maybe_last_update_user_token: Some("Last Update User Token".to_string()),
-        original_download_url: Some("https://civitai.com/models/97653/????".to_string()),
-        original_filename: Some("zVae_v20.safetensors".to_string()),
-        file_size_bytes: 0,
-        file_checksum_sha2: metadata6.sha256_checksum.to_string(),
-        public_bucket_hash: metadata6.bucket_details.clone().unwrap().object_hash,
-        maybe_public_bucket_prefix: Some(metadata6.bucket_details.clone().unwrap().prefix),
-        maybe_public_bucket_extension: Some(metadata6.bucket_details.clone().unwrap().suffix),
-        version: 1,
-        mysql_pool: &mysql_pool, // replace with actual MySqlPool
-    };
-
     create_weight(weights1).await?;
     create_weight(weights2).await?;
     create_weight(weights3).await?;
 
-
-    create_weight(weights4).await?;
-    create_weight(weights5).await?;
-    create_weight(weights6).await?;
-
     Ok(())
 }
 
 pub async fn seed_workflows_for_testing_inference(mysql_pool: &Pool<MySql>, user_token: UserToken) -> AnyhowResult<()>{
     let model_weight_token1 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token2 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token3 = ModelWeightToken::generate_for_testing_and_dev_seeding_never_use_in_production_seriously();
+    let model_weight_token4 = ModelWeightToken::new_from_str("weight_n8sz47gmfw2zx02snrbz88ns9");
 
     let mut path_to_comfy = get_seed_tool_data_root();
     path_to_comfy.push("models/workflows/comfyui/yae_video_prod.json");
@@ -727,6 +691,21 @@ pub async fn seed_workflows_for_testing_inference(mysql_pool: &Pool<MySql>, user
     let comfy_weights_descriptor = Box::new(WeightsWorkflowDescriptor {});
     let metadata1 = remote_cloud_file_client.upload_file(comfy_weights_descriptor, path_to_comfy.as_path().to_str().unwrap()).await?;
 
+    // let mut path_to_comfy2 = get_seed_tool_data_root();
+    // path_to_comfy2.push("models/workflows/comfyui/majicmixRealistic_v7.safetensors");
+    // let comfy_weights_descriptor2 = Box::new(WeightsSD15Descriptor {});
+    // let metadata2 = remote_cloud_file_client.upload_file(comfy_weights_descriptor2, path_to_comfy2.as_path().to_str().unwrap()).await?;
+
+    // let mut path_to_comfy3 = get_seed_tool_data_root();
+    // path_to_comfy3.push("models/workflows/comfyui/yae_video_test.json");
+    // let comfy_weights_descriptor3 = Box::new(WeightsWorkflowDescriptor {});
+    // let metadata3 = remote_cloud_file_client.upload_file(comfy_weights_descriptor3, path_to_comfy3.as_path().to_str().unwrap()).await?;
+
+/*    let mut path_to_comfy4 = get_seed_tool_data_root();
+    path_to_comfy4.push("models/workflows/comfyui/v3_sd15_mm.ckpt");
+    let comfy_weights_descriptor4 = Box::new(WeightsSD15CkptDescriptor {});
+    let metadata4 = remote_cloud_file_client.upload_file(comfy_weights_descriptor4, path_to_comfy4.as_path().to_str().unwrap()).await?;*/
+
     let weights1 = CreateModelWeightsArgs {
         token: &model_weight_token1, // replace with actual ModelWeightToken
         weights_type: WeightsType::ComfyUi, // replace with actual WeightsType
@@ -748,9 +727,76 @@ pub async fn seed_workflows_for_testing_inference(mysql_pool: &Pool<MySql>, user
         version: 1,
         mysql_pool: &mysql_pool, // replace with actual MySqlPool
     };
+    // let weights2 = CreateModelWeightsArgs {
+    //     token: &model_weight_token2, // replace with actual ModelWeightToken
+    //     weights_type: WeightsType::ComfyUi, // replace with actual WeightsType
+    //     weights_category: WeightsCategory::WorkflowConfig, // replace with actual WeightsCategory
+    //     title: "v1-5-pruned-emaonly".to_string(),
+    //     maybe_description_markdown: Some("Test model for ComfyUI".to_string()),
+    //     maybe_description_rendered_html: Some("<p>Description</p>".to_string()),
+    //     creator_user_token: Some(&user_token), // replace with actual UserToken
+    //     creator_ip_address: "192.168.1.1",
+    //     creator_set_visibility: Visibility::Public,
+    //     maybe_last_update_user_token: Some("Last Update User Token".to_string()),
+    //     original_download_url: Some("https://huggingface.co/runwayml/stable-diffusion-v1-5".to_string()),
+    //     original_filename: Some("v1-5-pruned-emaonly.ckpt".to_string()),
+    //     file_size_bytes: metadata2.file_size_bytes as i32,
+    //     file_checksum_sha2: metadata2.sha256_checksum.to_string(),
+    //     public_bucket_hash: metadata2.bucket_details.clone().unwrap().object_hash,
+    //     maybe_public_bucket_prefix: Some(metadata2.bucket_details.clone().unwrap().prefix),
+    //     maybe_public_bucket_extension: Some(metadata2.bucket_details.clone().unwrap().suffix),
+    //     version: 1,
+    //     mysql_pool: &mysql_pool, // replace with actual MySqlPool
+    // };
+
+    // let weights3 = CreateModelWeightsArgs {
+    //     token: &model_weight_token3, // replace with actual ModelWeightToken
+    //     weights_type: WeightsType::ComfyUi, // replace with actual WeightsType
+    //     weights_category: WeightsCategory::WorkflowConfig, // replace with actual WeightsCategory
+    //     title: "yae-video-test".to_string(),
+    //     maybe_description_markdown: Some("Test workflow for ComfyUI".to_string()),
+    //     maybe_description_rendered_html: Some("<p>Description</p>".to_string()),
+    //     creator_user_token: Some(&user_token), // replace with actual UserToken
+    //     creator_ip_address: "192.168.1.1",
+    //     creator_set_visibility: Visibility::Public,
+    //     maybe_last_update_user_token: Some("Last Update User Token".to_string()),
+    //     original_download_url: Some("https://github.com/comfyanonymous/ComfyUI".to_string()),
+    //     original_filename: Some("yae-video-test.json".to_string()),
+    //     file_size_bytes: metadata3.file_size_bytes as i32,
+    //     file_checksum_sha2: metadata3.sha256_checksum.to_string(),
+    //     public_bucket_hash: metadata3.bucket_details.clone().unwrap().object_hash,
+    //     maybe_public_bucket_prefix: Some(metadata3.bucket_details.clone().unwrap().prefix),
+    //     maybe_public_bucket_extension: Some(metadata3.bucket_details.clone().unwrap().suffix),
+    //     version: 1,
+    //     mysql_pool: &mysql_pool, // replace with actual MySqlPool
+    // };
+
+    // let weights4 = CreateModelWeightsArgs {
+    //     token: &model_weight_token4, // replace with actual ModelWeightToken
+    //     weights_type: WeightsType::ComfyUi, // replace with actual WeightsType
+    //     weights_category: WeightsCategory::WorkflowConfig, // replace with actual WeightsCategory
+    //     title: "v3_sd15_mm".to_string(),
+    //     maybe_description_markdown: Some("ComfyUI motion module".to_string()),
+    //     maybe_description_rendered_html: Some("<p>Description</p>".to_string()),
+    //     creator_user_token: Some(&user_token), // replace with actual UserToken
+    //     creator_ip_address: "192.168.1.1",
+    //     creator_set_visibility: Visibility::Public,
+    //     maybe_last_update_user_token: Some("Last Update User Token".to_string()),
+    //     original_download_url: Some("https://huggingface.co/guoyww/animatediff/blob/main/v3_sd15_mm.ckpt".to_string()),
+    //     original_filename: Some("v3_sd15_mm.ckpt".to_string()),
+    //     file_size_bytes: metadata4.file_size_bytes as i32,
+    //     file_checksum_sha2: metadata4.sha256_checksum.to_string(),
+    //     public_bucket_hash: metadata4.bucket_details.clone().unwrap().object_hash,
+    //     maybe_public_bucket_prefix: Some(metadata4.bucket_details.clone().unwrap().prefix),
+    //     maybe_public_bucket_extension: Some(metadata4.bucket_details.clone().unwrap().suffix),
+    //     version: 1,
+    //     mysql_pool: &mysql_pool, // replace with actual MySqlPool
+    // };
 
     create_weight(weights1).await?;
-
+    // create_weight(weights2).await?;
+    // create_weight(weights3).await?;
+    // create_weight(weights4).await?;
 
     Ok(())
 }
@@ -786,12 +832,12 @@ pub async fn seed_weights_for_user_token(
             file_size_bytes: 1024,
             file_checksum_sha2: "checksum1".to_string(),
             public_bucket_hash: "bucket_hash1".to_string(),
-            maybe_public_bucket_prefix: Some("_fake".to_string()),
-            maybe_public_bucket_extension: Some("rvc".to_string()),
-            
-            
-            
-            
+            maybe_public_bucket_prefix: Some("rvc_".to_string()),
+            maybe_public_bucket_extension: Some(".pt".to_string()),
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 1,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -813,10 +859,10 @@ pub async fn seed_weights_for_user_token(
             public_bucket_hash: "bucket_hash2".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("tt2".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 20,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 2,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 2,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -838,10 +884,10 @@ pub async fn seed_weights_for_user_token(
             public_bucket_hash: "bucket_hash3".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("safetensors".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 1,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -863,10 +909,10 @@ pub async fn seed_weights_for_user_token(
             public_bucket_hash: "bucket_hash4".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("LoRA".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 2,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -888,10 +934,10 @@ pub async fn seed_weights_for_user_token(
             public_bucket_hash: "bucket_hash4".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("LoRA".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 2,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         }
@@ -939,10 +985,10 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>, _user_token: UserTo
             public_bucket_hash: "bucket_hash1".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("rvc".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 1,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -964,10 +1010,10 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>, _user_token: UserTo
             public_bucket_hash: "bucket_hash2".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("tt2".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 20,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 2,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 2,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -989,10 +1035,10 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>, _user_token: UserTo
             public_bucket_hash: "bucket_hash3".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("safetensors".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 1,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -1014,10 +1060,10 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>, _user_token: UserTo
             public_bucket_hash: "bucket_hash4".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("LoRA".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 2,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         },
@@ -1039,10 +1085,10 @@ pub async fn original_seed_weights(mysql_pool: &Pool<MySql>, _user_token: UserTo
             public_bucket_hash: "bucket_hash5".to_string(),
             maybe_public_bucket_prefix: Some("_fake".to_string()),
             maybe_public_bucket_extension: Some("LoRA".to_string()),
-            
-            
-            
-            
+            //cached_user_ratings_total_count: 10,
+            //cached_user_ratings_positive_count: 9,
+            //cached_user_ratings_negative_count: 1,
+            //maybe_cached_user_ratings_ratio: Some(0.9),
             version: 2,
             mysql_pool: &mysql_pool, // replace with actual MySqlPool
         }
@@ -1071,7 +1117,7 @@ pub async fn seed_weights(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
     // seed_weights_for_paging(mysql_pool,user_token).await?;
     seed_weights_for_testing_inference(mysql_pool,user_token.clone()).await?;
     // println!("TESTING DOWLOAD");
-    // test_seed_weights_files().await?;
-    //seed_workflows_for_testing_inference(mysql_pool,user_token.clone()).await?;
+    //test_seed_weights_files().await?;
+    // seed_workflows_for_testing_inference(mysql_pool,user_token.clone()).await?;
     Ok(())
 }

crates/cli/elasticsearch-cli/src/cli_args.rs

@@ -33,7 +33,7 @@ pub enum Action {
 #[command(name="elasticsearch-cli")]
 pub struct Args {
   #[arg(name="action", long="action", help="action to take", required=true)]
-  action: String,
+  action: Action,
 
   #[arg(name="mysql", long="mysql", help="production or development")]
   mysql: Option<String>,
@@ -48,7 +48,7 @@ pub fn parse_cli_args() -> AnyhowResult<ParsedArgs> {
   Ok(ParsedArgs {
     mysql_environment: to_environment(args.mysql.as_deref())?,
     elasticsearch_environment: to_environment(args.elasticsearch.as_deref())?,
-    action: action_from_str(&args.action)?,
+    action: args.action,
   })
 }
 

crates/cli/elasticsearch-cli/src/plans/create_all_model_weight_documents.rs

@@ -53,6 +53,8 @@ pub async fn create_all_model_weight_documents(mysql: &Pool<MySql>, elasticsearc
 async fn create_document_from_record(elasticsearch: &Elasticsearch, record: ModelWeightForElasticsearchRecord) -> AnyhowResult<()> {
   info!("Create record for {:?} - {:?}", record.token, record.title);
 
+  let is_deleted = record.user_deleted_at.is_some() || record.mod_deleted_at.is_some();
+
   let document = ModelWeightDocument {
     token: record.token,
 
@@ -93,6 +95,8 @@ async fn create_document_from_record(elasticsearch: &Elasticsearch, record: Mode
     updated_at: record.updated_at,
     user_deleted_at: record.user_deleted_at,
     mod_deleted_at: record.mod_deleted_at,
+
+    is_deleted,
   };
 
   let op : BulkOperation<_> = BulkOperation::index(&document)

crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs

@@ -26,7 +26,7 @@ pub struct FileBucketDirectory {
       // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
       let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
       // gets you name of the file with suffix and prefix and entropy in the centre
-      let file_name = format!("{}_{}.{}", bucket_details.get_prefix(), entropy, bucket_details.get_suffix());
+      let file_name = format!("{}{}{}", bucket_details.get_prefix(), entropy, bucket_details.get_suffix());
       // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
       // This is the full path you download from
       let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name);
@@ -46,7 +46,7 @@ pub struct FileBucketDirectory {
       // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
       let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
       // gets you name of the file with suffix and prefix and entropy in the centre
-      let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix());
+      let file_name = format!("{}{}{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix());
       // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
 
       // This is the full path you upload to.

crates/lib/api_clients/cloud_storage/src/remote_file_manager/media_descriptor.rs

@@ -10,11 +10,11 @@ impl FileDescriptor for MediaImagePngDescriptor {
         MEDIA_FILE_DIRECTORY
     }
     fn get_suffix(&self)->String {
-        "png".to_string()
+        ".png".to_string()
     }
 
     fn get_prefix(&self)->String {
-        "image".to_string()
+        "image_".to_string()
     }
 
     fn is_public(&self) -> bool {
@@ -28,11 +28,11 @@ impl FileDescriptor for MediaVideoMp4Descriptor {
         MEDIA_FILE_DIRECTORY
     }
     fn get_suffix(&self)->String {
-        "mp4".to_string()
+        ".mp4".to_string()
     }
 
     fn get_prefix(&self)->String {
-        "video".to_string()
+        "video_".to_string()
     }
 
     fn is_public(&self) -> bool {
@@ -46,11 +46,11 @@ impl FileDescriptor for UploadVideoMp4Descriptor {
         MEDIA_FILE_DIRECTORY
     }
     fn get_suffix(&self)->String {
-        "mp4".to_string()
+        ".mp4".to_string()
     }
 
     fn get_prefix(&self)->String {
-        "upload".to_string()
+        "upload_".to_string()
     }
 
     fn is_public(&self) -> bool {

crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_bucket_details.rs

@@ -27,43 +27,44 @@ impl RemoteCloudBucketDetails {
         &self.suffix
     }
 
+    // this is not really useful for downloads remove...
     pub fn file_descriptor_from_bucket_details(&self) -> Box<dyn FileDescriptor> {
         match self.prefix.as_str() {
             // Weights
-            "loRA" => Box::new(weights_descriptor::WeightsLoRADescriptor {}),
-            "sd15" =>
+            "loRA_" => Box::new(weights_descriptor::WeightsLoRADescriptor {}),
+            "sd15_" =>
                 match self.suffix.as_str() {
-                    "safetensors" => Box::new(weights_descriptor::WeightsSD15Descriptor {}),
-                    "ckpt" => Box::new(weights_descriptor::WeightsSD15CkptDescriptor {}),
+                    ".safetensors" => Box::new(weights_descriptor::WeightsSD15Descriptor {}),
+                    ".ckpt" => Box::new(weights_descriptor::WeightsSD15CkptDescriptor {}),
                     _ => panic!("Unknown suffix: {}",self.suffix)
                 },
-            "sdxl" => Box::new(weights_descriptor::WeightsSDXLDescriptor {}),
-            "valle_prompt" => Box::new(weights_descriptor::WeightsVallePromptDescriptor {}),
-            "rvc" => {
+            "sdxl_" => Box::new(weights_descriptor::WeightsSDXLDescriptor {}),
+            "valle_prompt_" => Box::new(weights_descriptor::WeightsVallePromptDescriptor {}),
+            "rvc_" => {
                 match self.suffix.as_str() {
-                    "safetensors" => Box::new(weights_descriptor::WeightsRVCDescriptor {}),
-                    "index" => Box::new(weights_descriptor::WeightsRVCIndexDescriptor {}),
+                    ".safetensors" => Box::new(weights_descriptor::WeightsRVCDescriptor {}),
+                    ".index" => Box::new(weights_descriptor::WeightsRVCIndexDescriptor {}),
                     _ => panic!("Unknown suffix: {}",self.suffix)
                 }
             },
-            "svc" => Box::new(weights_descriptor::WeightsSVCDescriptor {}),
-            "workflow" => Box::new(weights_descriptor::WeightsWorkflowDescriptor {}),
+            "svc_" => Box::new(weights_descriptor::WeightsSVCDescriptor {}),
+            "workflow_" => Box::new(weights_descriptor::WeightsWorkflowDescriptor {}),
             // Media
-            "image" => {
+            "image_" => {
                 match self.suffix.as_str() {
-                    "png" => Box::new(media_descriptor::MediaImagePngDescriptor {}),
+                    ".png" => Box::new(media_descriptor::MediaImagePngDescriptor {}),
                     _ => panic!("Unknown suffix: {}",self.suffix)
                 }
             },
-            "video" => {
+            "video_" => {
                 match self.suffix.as_str() {
-                    "mp4" => Box::new(media_descriptor::MediaVideoMp4Descriptor {}),
+                    ".mp4" => Box::new(media_descriptor::MediaVideoMp4Descriptor {}),
                     _ => panic!("Unknown suffix: {}",self.suffix)
                 }
             },
-            "upload" => {
+            "upload_" => {
                 match self.suffix.as_str() {
-                    "mp4" => Box::new(media_descriptor::UploadVideoMp4Descriptor {}),
+                    ".mp4" => Box::new(media_descriptor::UploadVideoMp4Descriptor {}),
                     _ => panic!("Unknown suffix: {}",self.suffix)
                 }
             },

crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs

@@ -162,11 +162,11 @@ mod tests {
     #[tokio::test]
     async fn remote_file_manager_descriptor_test() {
         use super::*;
-        let remote_cloud_bucket_details = RemoteCloudBucketDetails::new("object_hash".to_string(), "loRA".to_string(), "safetensors".to_string());
+        let remote_cloud_bucket_details = RemoteCloudBucketDetails::new("object_hash".to_string(), "loRA_".to_string(), ".safetensors".to_string());
         let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details();
         
-        assert_eq!(file_descriptor.get_prefix(), "loRA");
-        assert_eq!(file_descriptor.get_suffix(), "safetensors");
+        assert_eq!(file_descriptor.get_prefix(), "loRA_");
+        assert_eq!(file_descriptor.get_suffix(), ".safetensors");
         assert_eq!(file_descriptor.is_public(), true);
 
     }
@@ -187,8 +187,8 @@ mod tests {
         println!("begin upload from file_path: {:?}", file_path);
         let details = RemoteCloudBucketDetails {
             object_hash: String::from("123"),
-            prefix: String::from("loRA"),
-            suffix: String::from("safetensors")
+            prefix: String::from("loRA_"),
+            suffix: String::from(".safetensors")
         };
         let result = remote_cloud_file_manager.download_file(details,file_path.to_string()).await;
         

crates/lib/api_clients/cloud_storage/src/remote_file_manager/weights_descriptor.rs

@@ -10,13 +10,13 @@ impl FileDescriptor for WeightsLoRADescriptor {
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-       "safetensors".to_string()
+       ".safetensors".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, loRA, sd15, sdxl when implmenting add to the end 
     fn get_prefix(&self)->String {
-        "loRA".to_string()
+        "loRA_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -35,13 +35,13 @@ impl FileDescriptor for WeightsSD15Descriptor {
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-       "safetensors".to_string()
+       ".safetensors".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end 
     fn get_prefix(&self)->String {
-        "sd15".to_string()
+        "sd15_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -59,13 +59,13 @@ impl FileDescriptor for WeightsSD15CkptDescriptor {
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-       "ckpt".to_string()
+       ".ckpt".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
     fn get_prefix(&self)->String {
-        "sd15".to_string()
+        "sd15_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -84,13 +84,13 @@ impl FileDescriptor for WeightsSDXLDescriptor {
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-       "safetensors".to_string()
+       ".safetensors".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end 
     fn get_prefix(&self)->String {
-        "sdxl".to_string()
+        "sdxl_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -108,13 +108,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsS
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-        "safetensors".to_string()
+        ".safetensors".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
     fn get_prefix(&self)->String {
-        "svc".to_string()
+        "svc_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -132,13 +132,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-        "pth".to_string()
+        ".pth".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
     fn get_prefix(&self)->String {
-        "rvc".to_string()
+        "rvc_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -156,13 +156,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-        "index".to_string()
+        ".index".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
     fn get_prefix(&self)->String {
-        "rvc".to_string()
+        "rvc_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -180,13 +180,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsV
     // this will be the type of file peroid is handled by the file formatter
     // e.g safetensors bin jpg
     fn get_suffix(&self)->String {
-        "safetensors".to_string()
+        ".safetensors".to_string()
     }
     // This will be the prefix of the media type or the weights type.
     // name of the weights or the name of the media type
     // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
     fn get_prefix(&self)->String {
-        "valle_prompt".to_string()
+        "valle_prompt_".to_string()
     }
 
     // This will be ensure that the right bucket is picked
@@ -203,11 +203,11 @@ impl FileDescriptor for WeightsWorkflowDescriptor {
     }
 
     fn get_suffix(&self) -> String {
-        "json".to_string()
+        ".json".to_string()
     }
 
     fn get_prefix(&self) -> String {
-        "workflow".to_string()
+        "workflow_".to_string()
     }
 
     fn is_public(&self) -> bool {

crates/lib/elasticsearch_schema/src/documents/model_weight_document.rs

@@ -58,6 +58,9 @@ pub struct ModelWeightDocument {
   pub updated_at: DateTime<Utc>,
   pub user_deleted_at: Option<DateTime<Utc>>,
   pub mod_deleted_at: Option<DateTime<Utc>>,
+
+  /// Calculated as "either user or mod deleted"
+  pub is_deleted: bool,
 }
 
 impl Document for ModelWeightDocument {

crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs

@@ -58,6 +58,10 @@ pub enum InferenceCategoryAbbreviated {
   #[serde(rename = "fc")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
   #[serde(alias = "format_conversion")]
   FormatConversion,
+
+  #[serde(rename = "bw")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(alias = "convert_bvh_to_workflow")]
+  ConvertBvhToWorkflow,
 }
 
 #[derive(Serialize, Deserialize, Clone, Copy, Debug)]
@@ -143,6 +147,7 @@ impl InferenceCategoryAbbreviated {
       InferenceCategory::Mocap => Self::Mocap,
       InferenceCategory::Workflow => Self::Workflow,
       InferenceCategory::FormatConversion => Self::FormatConversion,
+      InferenceCategory::ConvertBvhToWorkflow => Self::ConvertBvhToWorkflow,
     }
   }
 
@@ -156,6 +161,7 @@ impl InferenceCategoryAbbreviated {
       Self::Mocap => InferenceCategory::Mocap,
       Self::Workflow => InferenceCategory::Workflow,
       Self::FormatConversion => InferenceCategory::FormatConversion,
+      Self::ConvertBvhToWorkflow => InferenceCategory::ConvertBvhToWorkflow,
     }
   }
 }

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -121,6 +121,7 @@ SELECT
     jobs.on_success_result_entity_type as maybe_result_entity_type,
     jobs.on_success_result_entity_token as maybe_result_entity_token,
 
+    model_weights.title as maybe_model_weights_title,
     tts_models.title as maybe_tts_model_title,
     voice_conversion_models.title as maybe_voice_conversion_model_title,
 
@@ -144,6 +145,7 @@ SELECT
 
 FROM generic_inference_jobs as jobs
 
+LEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token
 LEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token
 LEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token
 
@@ -174,16 +176,25 @@ WHERE jobs.token = ?
 
 /// Map the internal record type (since we query over several result tables)
 fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericInferenceJobStatus {
-  let maybe_model_title = match record.inference_category {
-    InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
-    InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
-    InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
-    InferenceCategory::VideoFilter => Some("Video Filter"),
-    InferenceCategory::ImageGeneration => Some("Image Generation"),
-    InferenceCategory::Mocap => Some("Mocap"),
-    InferenceCategory::Workflow => Some("Workflow"),
-    InferenceCategory::FormatConversion => Some("format conversion"),
-  };
+  let mut maybe_model_title = None;
+
+  if let Some(title) = record.maybe_model_weights_title.as_deref() {
+    maybe_model_title = Some(title);
+  }
+
+  if maybe_model_title.is_none() {
+    maybe_model_title = match record.inference_category {
+      InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
+      InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
+      InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
+      InferenceCategory::VideoFilter => Some("Video Filter"),
+      InferenceCategory::ImageGeneration => Some("Image Generation"),
+      InferenceCategory::Mocap => Some("Mocap"),
+      InferenceCategory::Workflow => Some("Workflow"),
+      InferenceCategory::FormatConversion => Some("format conversion"),
+      InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
+    };
+  }
 
   // NB: A bit of a hack. We store TTS results with a full path.
   // Going forward, all other record types will store a hash.
@@ -196,6 +207,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
     InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
+    InferenceCategory::ConvertBvhToWorkflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
   };
 
   // NB: We've moved voice conversion out of their own table and into media_files
@@ -275,6 +287,7 @@ struct RawGenericInferenceJobStatus {
   pub maybe_result_entity_type: Option<String>,
   pub maybe_result_entity_token: Option<String>,
 
+  pub maybe_model_weights_title: Option<String>,
   pub maybe_tts_model_title: Option<String>,
   pub maybe_voice_conversion_model_title: Option<String>,
 

crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs

@@ -28,6 +28,12 @@ pub struct InsertGenericInferenceArgs<'a> {
   pub maybe_input_source_token: Option<&'a str>,
   pub maybe_input_source_token_type: Option<InferenceInputSourceTokenType>,
 
+  // For jobs that perform "downloads", this is the URL to download.
+  // NB: Some jobs aren't using this field yet and will pack the URL inside of
+  //   the "GenericInferenceArgs" field. The goal is to migrate them to this
+  //   top-level field eventually.
+  pub maybe_download_url: Option<&'a str>,
+
   pub maybe_raw_inference_text: Option<&'a str>,
 
   pub maybe_max_duration_seconds: Option<i32>,
@@ -83,6 +89,8 @@ SET
   maybe_input_source_token = ?,
   maybe_input_source_token_type = ?,
 
+  maybe_download_url = ?,
+
   maybe_raw_inference_text = ?,
 
   maybe_inference_args = ?,
@@ -115,6 +123,8 @@ SET
         args.maybe_input_source_token,
         args.maybe_input_source_token_type,
 
+        args.maybe_download_url,
+
         args.maybe_raw_inference_text,
 
         serialized_args_payload,

crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_from_comfy_ui.rs

@@ -32,6 +32,8 @@ pub struct InsertArgs<'a> {
     pub worker_hostname: &'a str,
     pub worker_cluster: &'a str,
     pub media_file_type: MediaFileType,
+
+    pub extra_file_modification_info: Option<&'a str>,
 }
 
 pub async fn insert_media_file_from_comfy_ui(args: InsertArgs<'_>) -> AnyhowResult<(MediaFileToken, u64)>
@@ -92,6 +94,7 @@ SET
   creator_ip_address = ?,
 
   creator_set_visibility = ?,
+  extra_file_modification_info = ?,
 
   maybe_creator_file_synthetic_id = ?,
   maybe_creator_category_synthetic_id = ?,
@@ -122,6 +125,7 @@ SET
       args.job.creator_ip_address,
 
       args.job.creator_set_visibility.to_str(),
+      args.extra_file_modification_info,
 
       maybe_creator_file_synthetic_id,
       maybe_creator_category_synthetic_id,

crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_from_tacotron2.rs

@@ -84,6 +84,8 @@ SET
   maybe_origin_model_type = ?,
   maybe_origin_model_token = ?,
 
+  maybe_text_transcript = ?,
+
   media_type = ?,
   maybe_mime_type = ?,
   file_size_bytes = ?,
@@ -116,6 +118,8 @@ SET
       origin_model_type.to_str(),
       args.job.maybe_model_token,
 
+      args.job.maybe_raw_inference_text,
+
       MEDIA_TYPE.to_str(),
       args.maybe_mime_type,
       args.file_size_bytes,

crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_from_zero_shot_tts.rs

@@ -81,8 +81,11 @@ pub async fn insert_media_file_from_zero_shot(
     
       origin_category = ?,
       origin_product_category = ?,
+
       maybe_origin_model_type = ?,
-    
+
+      maybe_text_transcript = ?,
+
       media_type = ?,
       maybe_mime_type = ?,
       file_size_bytes = ?,
@@ -113,6 +116,8 @@ pub async fn insert_media_file_from_zero_shot(
           ORIGIN_PRODUCT_CATEGORY.to_str(),
           args.origin_model_type.to_str(),
 
+          args.job.maybe_raw_inference_text,
+
           MEDIA_TYPE.to_str(),
           args.maybe_mime_type,
           args.file_size_bytes,

crates/lib/mysql_queries/src/queries/model_weights/get/get_weight.rs

@@ -33,7 +33,7 @@ pub struct RetrievedModelWeight {
     pub maybe_last_update_user_token: Option<UserToken>,
     pub original_download_url: Option<String>,
     pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
     pub file_checksum_sha2: String,
     pub public_bucket_hash: String,
     pub maybe_public_bucket_prefix: Option<String>,
@@ -271,7 +271,7 @@ pub struct RawWeight {
     pub maybe_last_update_user_token: Option<UserToken>,
     pub original_download_url: Option<String>,
     pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
     pub file_checksum_sha2: String,
     pub public_bucket_hash: String,
     pub maybe_public_bucket_prefix: Option<String>,

crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_inference.rs

@@ -32,7 +32,7 @@ pub struct ModelWeightForLegacyTtsInference {
     pub maybe_last_update_user_token: Option<UserToken>,
     pub original_download_url: Option<String>,
     pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
     pub file_checksum_sha2: String,
     pub public_bucket_hash: String,
     pub maybe_public_bucket_prefix: Option<String>,
@@ -260,7 +260,7 @@ pub struct RawWeight {
     maybe_last_update_user_token: Option<UserToken>,
     original_download_url: Option<String>,
     original_filename: Option<String>,
-    file_size_bytes: i32,
+    file_size_bytes: i64,
     file_checksum_sha2: String,
     public_bucket_hash: String,
     maybe_public_bucket_prefix: Option<String>,

crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_info.rs

@@ -31,7 +31,7 @@ pub struct ModelWeightForLegacyTtsInfo {
     pub maybe_last_update_user_token: Option<UserToken>,
     pub original_download_url: Option<String>,
     pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
     pub file_checksum_sha2: String,
     pub public_bucket_hash: String,
     pub maybe_public_bucket_prefix: Option<String>,
@@ -298,7 +298,7 @@ pub struct RawWeight {
     pub maybe_last_update_user_token: Option<UserToken>,
     pub original_download_url: Option<String>,
     pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
     pub file_checksum_sha2: String,
     pub public_bucket_hash: String,
     pub maybe_public_bucket_prefix: Option<String>,

crates/lib/mysql_queries/src/queries/users/user_bookmarks/list_user_bookmarks.rs

@@ -55,7 +55,8 @@ fn select_result_fields() -> &'static str {
     entity_stats.bookmark_count as maybe_bookmark_count,
 
     media_files.media_type as maybe_media_file_type,
-    media_files.origin_product_category as maybe_media_file_origin_category,
+    media_files.origin_category as maybe_media_file_origin_category,
+    media_files.origin_product_category as maybe_media_file_origin_product,
     media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_hash,
     media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,
     media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,

crates/lib/mysql_queries/src/queries/users/user_bookmarks/list_user_bookmarks_result.rs

@@ -3,6 +3,7 @@ use sqlx::{FromRow, Row};
 use sqlx::mysql::MySqlRow;
 
 use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
 use enums::by_table::media_files::media_file_type::MediaFileType;
 use enums::by_table::model_weights::weights_category::WeightsCategory;
 use enums::by_table::model_weights::weights_types::WeightsType;
@@ -93,6 +94,7 @@ pub struct RawUserBookmarkRecord {
 
   pub(crate) maybe_media_file_type: Option<MediaFileType>,
   pub(crate) maybe_media_file_origin_category: Option<MediaFileOriginCategory>,
+  pub(crate) maybe_media_file_origin_product: Option<MediaFileOriginProductCategory>,
   pub(crate) maybe_media_file_public_bucket_hash: Option<String>,
   pub(crate) maybe_media_file_public_bucket_prefix: Option<String>,
   pub(crate) maybe_media_file_public_bucket_extension: Option<String>,
@@ -199,6 +201,7 @@ impl FromRow<'_, MySqlRow> for RawUserBookmarkRecord {
         maybe_bookmark_count: row.try_get("maybe_bookmark_count")?,
         maybe_media_file_type: MediaFileType::try_from_mysql_row_nullable(row, "maybe_media_file_type")?,
         maybe_media_file_origin_category: MediaFileOriginCategory::try_from_mysql_row_nullable(row,"maybe_media_file_origin_category")?,
+        maybe_media_file_origin_product: MediaFileOriginProductCategory::try_from_mysql_row_nullable(row,"maybe_media_file_origin_product")?,
         maybe_media_file_public_bucket_hash: row.try_get("maybe_media_file_public_bucket_hash")?,
         maybe_media_file_public_bucket_prefix: row.try_get("maybe_media_file_public_bucket_prefix")?,
         maybe_media_file_public_bucket_extension: row.try_get("maybe_media_file_public_bucket_extension")?,

crates/lib/mysql_queries/src/queries/users/user_sessions/get_user_session_by_token.rs

@@ -37,6 +37,10 @@ pub struct SessionUserRecord {
   pub preferred_w2l_result_visibility: Visibility,
   pub auto_play_video_preference: Option<bool>,
 
+  // ===== FEATURE FLAGS ===== //
+
+  pub can_access_studio: bool,
+
   // ===== ROLE ===== //
 
   pub user_role_slug: String,
@@ -116,6 +120,8 @@ SELECT
     users.user_role_slug,
     users.is_banned,
 
+    users.can_access_studio,
+
     user_roles.can_use_tts,
     user_roles.can_use_w2l,
     user_roles.can_delete_own_tts_results,
@@ -176,6 +182,8 @@ WHERE user_sessions.token = ?
 
         is_banned: i8_to_bool(raw_user_record.is_banned),
 
+        can_access_studio: i8_to_bool(raw_user_record.can_access_studio),
+
         // Usage
         can_use_tts: nullable_i8_to_bool_default_false(raw_user_record.can_use_tts),
         can_use_w2l: nullable_i8_to_bool_default_false(raw_user_record.can_use_w2l),
@@ -239,6 +247,9 @@ struct SessionUserRawDbRecord {
   user_role_slug: String,
   is_banned: i8,
 
+  // Feature / Rollout Flags
+  can_access_studio: i8,
+
   // NB: These are `Option` due to the JOIN not being compile-time assured.
   // Usage
   can_use_tts: Option<i8>,

crates/service/job/es-update-job/src/main_loop.rs

@@ -106,6 +106,8 @@ pub async fn with_database_main_loop(updated_at_cursor: &mut DateTime<Utc>, job_
 async fn create_document_from_record(elasticsearch: &Elasticsearch, record: ModelWeightForElasticsearchRecord) -> AnyhowResult<()> {
   info!("Create record for {:?} - {:?}", record.token, record.title);
 
+  let is_deleted = record.user_deleted_at.is_some() || record.mod_deleted_at.is_some();
+
   let document = ModelWeightDocument {
     token: record.token,
 
@@ -146,6 +148,8 @@ async fn create_document_from_record(elasticsearch: &Elasticsearch, record: Mode
     updated_at: record.updated_at,
     user_deleted_at: record.user_deleted_at,
     mod_deleted_at: record.mod_deleted_at,
+
+    is_deleted,
   };
 
   let op : BulkOperation<_> = BulkOperation::index(&document)

crates/service/job/inference_job/Cargo.toml

@@ -84,4 +84,7 @@ mime = "0.3.17"
 rand = { version = "0.8.5", features = [] }
 jsonpath_lib = "0.3.0"
 walkdir = "2.4.0"
-reqwest = "0.11.22"
+reqwest = "0.11.22"
+opentelemetry = {version = "0.21.0", features = ["metrics"]}
+opentelemetry-otlp = { version = "0.14.0", features = ["metrics"] }
+opentelemetry_sdk = { version = "0.21.2", features = ["metrics", "rt-tokio"] }

crates/service/job/inference_job/config/inference-job.development.env

@@ -9,6 +9,9 @@ HTTP_BIND_ADDRESS=0.0.0.0:11223
 # NB: Disable if not testing Redis
 REDIS_FOR_KEEPALIVE_URL="redis://localhost/0"
 
+# General model weights cache directory
+MODEL_WEIGHTS_CACHE_DIR="/tmp/model_weights_cache"
+
 # ================================= #
 #                                   #
 #            Vall-E-X               #
@@ -28,9 +31,16 @@ VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .venv/bin/activate"
 #VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .env/bin/activate"
 #VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE="" # fill this out for dev?
 
-STYLE_TTS2_INFERENCE_ROOT_DIRECTORY=""
-STYLE_TTS2_INFERENCE_COMMAND = ""
-STYLE_TTS2_INFERENCE_EXECUTABLE = ""
+# ================================= #
+#                                   #
+#           STYLE TTS 2             #
+#                                   #
+# ================================= #
+
+# TODO(bt,2024-02-10): Not sure what these should be
+STYLE_TTS2_INFERENCE_EXECUTABLE=""
+STYLE_TTS2_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/tts/style-tts2"
+STYLE_TTS2_INFERENCE_COMMAND="venv/bin/python3.8 main.py"
 
 # ================================= #
 #                                   #
@@ -104,6 +114,7 @@ VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b"
 #            Rerender               #
 #                                   #
 # ================================= #
+
 RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/"
 RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py"
 
@@ -112,12 +123,15 @@ RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py"
 #          STABLE DIFFUSION         #
 #                                   #
 # ================================= #
+
 STABLE_DIFFUSION_INFERENCE_ROOT_DIRECTORY="/home/tensor/code/storyteller/storyteller-ml/image_generation/Storyteller-Studio-Filter/"
 STABLE_DIFFUSION_INFERENCE_COMMAND=".venv/bin/python main.py"
-SD_VAE_BUCKET_PATH="/home/tensor/code/storyteller/seed-tool-data/models/imagegen/vae/vaeFtMse840000EmaPruned_vae.safetensors"
-SD_PREDEFINED_SD_WEIGHT_TOKEN=""
-SD_PREDEFINED_LORA_WEIGHT_TOKEN=""
+SD_VAE_BUCKET_PATH="/stable_diffusion_deps/vaeFtEma560000Ema_original.pt"
+SD_PREDEFINED_SD_WEIGHT_BUCKET_PATH="/stable_diffusion_deps/fiamixRebootPastel_v14LightColor.safetensors"
 
+# TODO(bt,2024-02-10): Remove these
+SD_PREDEFINED_SD_WEIGHT_TOKEN="INVALID_VALUE_REMOVE"
+SD_PREDEFINED_LORA_WEIGHT_TOKEN="INVALID_VALUE_REMOVE"
 
 # ================================= #
 #                                   #
@@ -134,18 +148,30 @@ MOCAPNET_INFERENCE_EXECUTABLE_OR_COMMAND="pythonVirtualEnvironment/bin/python -m
 # ================================= #
 COMFY_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/image_generation/ComfyUI"
 COMFY_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python ../ComfyLauncher/ComfyRunner.py"
-STYLE_TTS2_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/tts/style-tts2"
-STYLE_TTS2_INFERENCE_COMMAND="venv/bin/python3.8 main.py"
+
+# TODO(bt,2024-02-10): Not sure what these should be
+COMFY_MOUNTS_DIRECTORY="NOT_SURE_YET"
+COMFY_WORKFLOW_BUCKET_PATH="NOT_SURE_YET"
 
 # ================================= #
 #                                   #
 #            Fbx2Gltf               #
 #                                   #
 # ================================= #
+
 #FBX2GLTF_DIRECTORY="%storyteller_root%/FBX2glTF"
 FBX2GLTF_DIRECTORY="/home/bt/dev/storyteller/FBX2glTF"
 FBX2GLTF_EXECUTABLE_OR_COMMAND="./bin/FBX2glTF-linux-x64"
 
+# ================================= #
+#                                   #
+#         BEVY COMPOSITOR           #
+#                                   #
+# ================================= #
+
+# TODO(bt,2024-02-10): Not sure what these should be
+BVH2WORKFLOW_EXECUTABLE_OR_COMMAND="NOT_SURE_YET"
+
 # ================================= #
 #                                   #
 #          CACHE SETTINGS           #

crates/service/job/inference_job/src/job/job_loop/main_loop.rs

@@ -9,13 +9,17 @@ use jobs_common::noop_logger::NoOpLogger;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::{AvailableInferenceJob, list_available_generic_inference_jobs, ListAvailableGenericInferenceJobArgs};
 use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed;
 use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_failure::mark_generic_inference_job_failure;
+use opentelemetry::metrics::AsyncInstrument;
 
+use crate::OTEL_METER_NAME;
 use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem;
 use crate::job::job_loop::process_single_job::process_single_job;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
 use crate::job_dependencies::JobDependencies;
 
+use opentelemetry::{global as otel, KeyValue as OtelAttribute, metrics::Unit};
+
 // Job runner timeouts (guards MySQL)
 const START_TIMEOUT_MILLIS : u64 = 500;
 const INCREASE_TIMEOUT_MILLIS : u64 = 1000;
@@ -61,6 +65,7 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
     }).await;
 
     let batch_query_duration = Instant::now().duration_since(batch_query_start_time);
+    job_dependencies.job_instruments.batch_query_duration.record(batch_query_duration.as_millis() as u64, &[]);
 
     if batch_query_duration > SLOW_BATCH_QUERY_NOTICE_DURATION {
       warn!("Batch query took duration: {:?}", &batch_query_duration);
@@ -91,10 +96,16 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
       continue;
     }
 
+    job_dependencies.job_instruments.batch_size.record(jobs.len() as u64, &[]);
     info!("Queried {} jobs from database", jobs.len());
 
+    let batch_processing_start_time = Instant::now();
+
     let batch_result = process_job_batch(&job_dependencies, jobs).await;
 
+    let batch_processing_duration = Instant::now().duration_since(batch_processing_start_time);
+    job_dependencies.job_instruments.batch_processing_duration.record(batch_processing_duration.as_millis() as u64, &[]);
+
     match batch_result {
       Ok(_) => {},
       Err(e) => {
@@ -118,9 +129,14 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
 
 async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<AvailableInferenceJob>) -> AnyhowResult<()> {
   let job_count = jobs.len();
+
   for (i, job) in jobs.into_iter().enumerate() {
+    job_dependencies.job_instruments.total_job_count.add(1, &[]);
+
+    let start_time = Instant::now();
     let result = process_single_job(job_dependencies, &job).await;
 
+
     match result {
       Ok(success_case) => {
         info!("Job loop iteration ({i} of {job_count} batch) \"success\": {:?}", success_case);
@@ -134,6 +150,7 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
 
         if increment_success_count {
           let stats = job_dependencies.job.info.job_stats.increment_success_count().ok();
+          job_dependencies.job_instruments.job_success_count.add(1, &[]);
           warn!("Success stats: {:?}", stats);
         }
       },
@@ -145,6 +162,21 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
         let _r = handle_error(&job_dependencies, &job, err).await?;
       }
     }
+    let job_duration = Instant::now().duration_since(start_time);;
+
+    let model_type_str = match job.maybe_model_type {
+      Some(model_type) => model_type.to_string(),
+      None => "unknown".to_string(),
+    };
+
+    job_dependencies.job_instruments.job_duration.record(job_duration.as_millis() as u64,
+      &[
+        OtelAttribute::new("job_user_is_premium", job.is_from_premium_user),
+        OtelAttribute::new("job_model", model_type_str),
+        OtelAttribute::new("job_status", job.status.to_str()),
+        OtelAttribute::new("job_inference_category", job.inference_category.to_str()),
+      ]
+    );
   }
 
   Ok(())
@@ -256,6 +288,10 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere
     warn!("Failure stats: {:?}", stats);
   }
 
+  job_dependencies.job_instruments.job_failure_count.add(1, &[
+    OtelAttribute::new("job_failure_internal_reason", internal_failure_reason.clone()),
+  ]);
+
   match job_failure_class {
     JobFailureClass::PermanentFailure => {
       let _r = mark_generic_inference_job_completely_failed(

crates/service/job/inference_job/src/job/job_loop/process_single_job.rs

@@ -13,6 +13,7 @@ use redis_common::redis_keys::RedisKeys;
 use crate::job::job_loop::determine_dependency_status::determine_dependency_status;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
+use crate::job::job_types::bevy_to_workflow::process_single_bevy_to_workflow_conversion_job::process_single_bevy_to_workflow_conversion_job;
 use crate::job::job_types::format_conversion::process_single_format_conversion_job::process_single_format_conversion_job;
 use crate::job::job_types::image_generation::process_single_ig_job::process_single_ig_job;
 use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
@@ -202,6 +203,9 @@ async fn do_process_single_job(
     InferenceCategory::FormatConversion => {
       process_single_format_conversion_job(job_dependencies, job).await?
     }
+    InferenceCategory::ConvertBvhToWorkflow => {
+      process_single_bevy_to_workflow_conversion_job(job_dependencies, job).await?
+    }
   };
 
   let maybe_entity_type = job_success_result.maybe_result_entity
@@ -234,6 +238,8 @@ async fn do_process_single_job(
 
   info!("Saved model record: {} - {}", job.id.0, &job.inference_job_token);
 
+  job_dependencies.job_instruments.inference_command_execution_duration.record(inference_duration.as_millis() as u64, &[]);
+
   // TODO(bt, 2023-01-11): Need to publish that the job finished.
   //  Publish the *correct type* of event.
   //job_dependencies.firehose_publisher.publish_generic_download_finished(

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/command_args.rs

@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use filesys::path_to_string::path_to_string;
+use subprocess_common::command_runner::command_args::CommandArgs;
+
+pub struct BvhToWorkflowCommandArgs<'a> {
+    pub input_file: &'a Path,
+    pub output_directory: &'a Path,
+}
+
+impl CommandArgs for BvhToWorkflowCommandArgs<'_> {
+    fn to_command_string(&self) -> String {
+        let mut command = String::new();
+
+        command.push_str(" -i ");
+        command.push_str(&path_to_string(self.input_file));
+
+        command.push_str(" -o ");
+        command.push_str(&path_to_string(self.output_directory));
+
+        command
+    }
+}

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/dependencies.rs

@@ -0,0 +1,46 @@
+use std::path::PathBuf;
+
+use errors::AnyhowResult;
+use subprocess_common::command_runner::command_runner::CommandRunner;
+use subprocess_common::command_runner::env_var_policy::EnvVarPolicy;
+use subprocess_common::docker_options::{DockerFilesystemMount, DockerOptions};
+use subprocess_common::executable_or_command::ExecutableOrShellCommand;
+
+pub struct BvhToWorkflowDependencies {
+    pub command_runner: CommandRunner,
+}
+
+impl BvhToWorkflowDependencies {
+    pub fn setup() -> AnyhowResult<Self> {
+        let maybe_execution_directory = easyenv::get_env_pathbuf_optional("BVH2WORKFLOW_DIRECTORY");
+
+        let executable_or_command = easyenv::get_env_string_required("BVH2WORKFLOW_EXECUTABLE_OR_COMMAND")?;
+
+        let executable_or_command = if executable_or_command.contains(" ") {
+            ExecutableOrShellCommand::BashShellCommand(executable_or_command)
+        } else {
+            ExecutableOrShellCommand::Executable(PathBuf::from(executable_or_command))
+        };
+
+        let maybe_execution_timeout = easyenv::get_env_duration_seconds_optional("BVH2WORKFLOW_TIMEOUT_SECONDS");
+
+        let maybe_docker_options = easyenv::get_env_string_optional("BVH2WORKFLOW_DOCKER_IMAGE")
+            .map(|image_name| DockerOptions {
+                image_name,
+                maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+                maybe_environment_variables: None,
+                maybe_gpu: None,
+            });
+
+        Ok(Self {
+            command_runner: CommandRunner {
+                executable_or_command,
+                maybe_execution_directory,
+                env_var_policy: EnvVarPolicy::CopyNone,
+                maybe_virtual_env_activation_command: None,
+                maybe_docker_options,
+                maybe_execution_timeout,
+            }
+        })
+    }
+}

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/mod.rs

@@ -0,0 +1,3 @@
+pub mod process_job;
+pub mod command_args;
+pub mod dependencies;

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/process_job.rs

@@ -0,0 +1,290 @@
+use std::fs::read_to_string;
+use std::time::Instant;
+
+use anyhow::anyhow;
+use log::{error, info, warn};
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
+
+use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use filesys::check_file_exists::check_file_exists;
+use filesys::file_size::file_size;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
+use hashing::sha256::sha256_hash_file::sha256_hash_file;
+use mimetypes::mimetype_for_file::get_mimetype_for_file;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::{insert_media_file_generic, InsertArgs};
+use mysql_queries::queries::media_files::get_media_file_for_inference::MediaFileForInference;
+use subprocess_common::command_runner::command_runner_args::{FileOrCreate, RunAsSubprocessArgs};
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::command_args::BvhToWorkflowCommandArgs;
+use crate::job_dependencies::JobDependencies;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
+
+const BUCKET_FILE_EXTENSION : &str = ".mp4";
+const BUCKET_FRAMES_FILE_EXTENSION : &str = ".zip";
+
+pub struct BvhToWorkflowJobArgs<'a> {
+  pub job_dependencies: &'a JobDependencies,
+  pub job: &'a AvailableInferenceJob,
+  pub media_file: &'a MediaFileForInference,
+}
+
+pub async fn process_job(args: BvhToWorkflowJobArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
+  let job = args.job;
+  let media_file = args.media_file;
+
+  let mut job_progress_reporter = args.job_dependencies
+      .clients
+      .job_progress_reporter
+      .new_generic_inference(job.inference_job_token.as_str())
+      .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
+
+  let model_dependencies = args
+      .job_dependencies
+      .job
+      .job_specific_dependencies
+      .maybe_convert_bvh_to_workflow_dependencies
+      .as_ref()
+      .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("missing BvhToWorkflow dependencies".to_string())))?;
+
+  // ==================== TEMP DIR ==================== //
+
+  let work_temp_dir = format!("temp_file_convert_{}", job.id.0);
+
+  // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
+  let work_temp_dir = args.job_dependencies
+      .fs
+      .scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  // ==================== DOWNLOAD MEDIA FILE ==================== //
+
+  info!("Download media for conversion...");
+
+  let original_media_upload_fs_path = {
+    let original_media_file_fs_path = work_temp_dir.path().join("original.bvh");
+
+    let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
+      &media_file.public_bucket_directory_hash,
+      media_file.maybe_public_bucket_prefix.as_deref(),
+      media_file.maybe_public_bucket_extension.as_deref());
+
+    let bucket_object_path = media_file_bucket_path.to_full_object_pathbuf();
+
+    info!("Downloading media to bucket path: {:?}", &bucket_object_path);
+
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "media upload (original file)",
+      final_filesystem_file_path: &original_media_file_fs_path,
+      bucket_object_path: &bucket_object_path,
+      bucket_client: &args.job_dependencies.buckets.public_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;
+
+    original_media_file_fs_path
+  };
+
+  // ==================== SETUP FOR CONVERSION ==================== //
+
+  job_progress_reporter.log_status("running conversion")
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+
+  // ==================== RUN INFERENCE SCRIPT ==================== //
+
+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+  let output_directory = work_temp_dir.path().join("output");
+  let output_directory_actual = work_temp_dir.path();
+
+  let execution_start_time = Instant::now();
+
+  let command_exit_status = {
+    model_dependencies
+        .command_runner
+        .run_with_subprocess(RunAsSubprocessArgs {
+          args: Box::new(&BvhToWorkflowCommandArgs {
+            input_file: &original_media_upload_fs_path,
+            output_directory: &output_directory,
+          }),
+          maybe_stderr_output_file: Some(FileOrCreate::NewFileWithName(&stderr_output_file)),
+          maybe_stdout_output_file: None,
+        })
+  };
+
+  let execution_duration = Instant::now().duration_since(execution_start_time);
+
+  info!("Execution took duration to complete: {:?}", &execution_duration);
+
+  if !command_exit_status.is_success() {
+    error!("Execution failed: {:?}", command_exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+    }
+
+    safe_delete_temp_file(&original_media_upload_fs_path);
+    safe_delete_temp_directory(&output_directory_actual);
+    safe_delete_temp_directory(&work_temp_dir);
+
+    return Err(error);
+  }
+
+  // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
+
+  let output_video_file = output_directory_actual.join("output.mp4");
+  let output_frames_zip_file = output_directory_actual.join("output.zip");
+  // NB: The actual name of the file will be this:
+
+  info!("Checking that output files exist...");
+
+  check_file_exists(&output_video_file).map_err(|e| ProcessSingleJobError::Other(e))?;
+  check_file_exists(&output_frames_zip_file).map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!("Interrogating result file properties...");
+
+  let file_size_bytes = file_size(&output_video_file)
+      .map_err(|err| ProcessSingleJobError::Other(err))?;
+
+  let maybe_mimetype = get_mimetype_for_file(&output_video_file)
+      .map_err(|err| ProcessSingleJobError::from_io_error(err))?
+      .map(|mime| mime.to_string());
+
+  info!("Calculating sha256...");
+
+  let file_checksum = sha256_hash_file(&output_video_file)
+      .map_err(|err| {
+        ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err))
+      })?;
+
+  // ==================== UPLOAD AUDIO TO BUCKET ==================== //
+
+  job_progress_reporter.log_status("uploading result")
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  let result_video_bucket_location = MediaFileBucketPath::generate_new(
+    None,
+    Some(BUCKET_FILE_EXTENSION));
+
+  let result_bucket_object_pathbuf = result_video_bucket_location.to_full_object_pathbuf();
+
+  info!("Destination bucket path: {:?}", &result_bucket_object_pathbuf);
+
+  info!("Uploading media file...");
+
+  args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type(
+    &result_bucket_object_pathbuf,
+    &output_video_file,
+    "video/mp4")
+      .await
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  let result_zip_bucket_location = MediaFileBucketPath::from_object_hash(&result_video_bucket_location.get_object_hash(),
+    None,
+    Some(".zip"));
+
+  let result_bucket_object_pathbuf = result_zip_bucket_location.to_full_object_pathbuf();
+
+  info!("Destination bucket path: {:?}", &result_bucket_object_pathbuf);
+
+  info!("Uploading media file...");
+
+  args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type(
+    &result_bucket_object_pathbuf,
+    &output_frames_zip_file,
+  "application/zip")
+      .await
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  safe_delete_temp_file(&output_video_file);
+  safe_delete_temp_file(&output_frames_zip_file);
+  safe_delete_temp_directory(&output_directory_actual);
+
+  // ==================== DELETE DOWNLOADED FILE ==================== //
+
+  // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk...
+  safe_delete_temp_directory(&work_temp_dir);
+
+  // ==================== SAVE RECORDS ==================== //
+
+  info!("Saving record...");
+
+  let (inference_result_token, id) = insert_media_file_generic(InsertArgs {
+    pool: &args.job_dependencies.db.mysql_pool,
+    job: &job,
+    media_type: MediaFileType::Video,
+    origin_category: MediaFileOriginCategory::Processed,
+    origin_product_category: MediaFileOriginProductCategory::Mocap,
+    maybe_origin_model_type: None,
+    maybe_origin_model_token: None,
+    maybe_origin_filename: None,
+    maybe_mime_type: maybe_mimetype.as_deref(),
+    file_size_bytes,
+    maybe_duration_millis: None,
+    maybe_audio_encoding: None,
+    maybe_video_encoding: None,
+    maybe_frame_width: None,
+    maybe_frame_height: None,
+    public_bucket_directory_hash: result_video_bucket_location.get_object_hash(),
+    maybe_public_bucket_prefix: result_video_bucket_location.get_optional_prefix(),
+    maybe_public_bucket_extension: result_video_bucket_location.get_optional_extension(),
+    extra_file_modification_info: None,
+    maybe_creator_file_synthetic_id_category: IdCategory::MediaFile,
+    maybe_creator_category_synthetic_id_category: IdCategory::MocapResult,
+    maybe_mod_user_token: None,
+    is_batch_generated: false,
+    is_generated_on_prem: false,
+    checksum_sha2: &file_checksum,
+    generated_by_worker: Some(&args.job_dependencies.job.info.container.hostname),
+    generated_by_cluster: Some(&args.job_dependencies.job.info.container.cluster_name),
+  })
+      .await
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!("VC Done.");
+
+  //args.job_dependencies.clients.firehose_publisher.vc_inference_finished(
+  //  maybe_user_token.as_ref(),
+  //  &job.inference_job_token,
+  //  inference_result_token.as_str())
+  //    .await
+  //    .map_err(|e| {
+  //      error!("error publishing event: {:?}", e);
+  //      ProcessSingleJobError::Other(anyhow!("error publishing event"))
+  //    })?;
+
+  job_progress_reporter.log_status("done")
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!("Job {:?} complete success! Downloaded, executed successfully, and uploaded. Saved record: {}, Result Token: {}",
+        job.id, id, &inference_result_token);
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::MediaFile,
+      entity_token: inference_result_token.to_string(),
+    }),
+    inference_duration: execution_duration,
+  })
+}
+
+#[derive(Deserialize, Default)]
+struct FileMetadata {
+  pub duration_millis: Option<u64>,
+  pub mimetype: Option<String>,
+  pub file_size_bytes: u64,
+}

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/mod.rs

@@ -0,0 +1,2 @@
+pub mod bvh_to_workflow;
+pub mod process_single_bevy_to_workflow_conversion_job;

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/process_single_bevy_to_workflow_conversion_job.rs

@@ -0,0 +1,50 @@
+use anyhow::anyhow;
+use log::error;
+
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::get_media_file_for_inference::get_media_file_for_inference;
+use tokens::tokens::media_files::MediaFileToken;
+
+use crate::job::job_loop::job_success_result::JobSuccessResult;
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::process_job::BvhToWorkflowJobArgs;
+use crate::job_dependencies::JobDependencies;
+
+
+pub async fn process_single_bevy_to_workflow_conversion_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let maybe_media_file_token = job.maybe_input_source_token
+      .as_deref()
+      .map(|token| MediaFileToken::new_from_str(token));
+
+  let media_file_token = match maybe_media_file_token {
+    None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media file for format conversion job: {:?}", job.inference_job_token))),
+    Some(token) => token,
+  };
+
+  let maybe_media_file_result =
+      get_media_file_for_inference(&media_file_token, &job_dependencies.db.mysql_pool).await;
+
+  let media_file = match maybe_media_file_result {
+    Ok(Some(media_file)) => media_file,
+    Ok(None) => {
+      error!("no media file record found for token: {:?}", media_file_token);
+      return Err(ProcessSingleJobError::Other(anyhow!("no media file record found for token: {:?}", media_file_token)));
+    }
+    Err(err) => {
+      error!("error fetching media file record from db: {:?}", err);
+      return Err(ProcessSingleJobError::Other(err));
+    }
+  };
+
+  let job_success_result = bvh_to_workflow::process_job::process_job(BvhToWorkflowJobArgs {
+        job_dependencies,
+        job,
+        media_file: &media_file,
+      }).await?;
+
+  Ok(job_success_result)
+}
+
+

crates/service/job/inference_job/src/job/job_types/image_generation/sd/mod.rs

@@ -3,4 +3,7 @@ pub mod process_job;
 pub mod model_downloaders;
 pub mod sd_inference_command;
 pub mod stable_diffusion_dependencies;
-pub mod validate_inputs;
+pub mod validate_inputs;
+mod process_job_inference;
+mod process_job_lora_upload;
+mod process_job_sd_upload;

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job.rs

@@ -1,52 +1,18 @@
-use std::fs::read_to_string;
-use std::path::PathBuf;
-use std::time::{Duration, Instant};
-
 use actix_web::dev::ResourcePath;
 use anyhow::anyhow;
-use log::{error, info, warn};
 use serde_json;
 
-use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
-use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
-use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
-use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
-use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
-use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
-use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
-use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
-use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
-use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
-use enums::by_table::media_files::media_file_type::MediaFileType;
-use enums::by_table::model_weights::weights_category::WeightsCategory;
-use enums::by_table::model_weights::weights_types::WeightsType;
-use filesys::file_exists::file_exists;
-use filesys::path_to_string::path_to_string;
-use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
-use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
 use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
-use mysql_queries::queries::batch_generations::insert_batch_generation_records::insert_batch_generation_records;
-use mysql_queries::queries::batch_generations::insert_batch_generation_records::InsertBatchArgs;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::media_files::create::insert_media_file_generic::insert_media_file_generic;
-use mysql_queries::queries::media_files::create::insert_media_file_generic::InsertArgs;
-use mysql_queries::queries::model_weights::create::create_weight::{
-    create_weight,
-    CreateModelWeightsArgs,
-};
-use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
-use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
-use tokens::tokens::model_weights::ModelWeightToken;
-use tokens::tokens::users::UserToken;
 
-use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
-use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job::job_types::image_generation::sd::process_job_inference::process_job_inference;
+use crate::job::job_types::image_generation::sd::process_job_lora_upload::process_job_lora_upload;
+use crate::job::job_types::image_generation::sd::process_job_sd_upload::process_job_sd_upload;
 use crate::job_dependencies::JobDependencies;
 
-const SD_IMAGE_PREFIX : &str = "image_";
-const SD_IMAGE_SUFFIX : &str = ".png";
-
 #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
 pub struct InferenceValues {
     pub prompt: String,
@@ -73,11 +39,7 @@ pub async fn sd_args_from_job(
 ) -> Result<StableDiffusionArgs, ProcessSingleJobError> {
     let inference_args = args.job.maybe_inference_args
         .as_ref()
-        .map(
-            |
-                args: &mysql_queries::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs
-            | args.args.as_ref()
-        )
+        .map(|args: &GenericInferenceArgs| args.args.as_ref())
         .flatten();
 
     let polymorphic_args = match inference_args {
@@ -111,874 +73,14 @@ pub async fn process_job_selection(
     if sd_args.type_of_inference == "inference" {
         process_job_inference(&args).await
     } else if sd_args.type_of_inference == "lora" {
-        process_job_lora(&args).await
+        process_job_lora_upload(&args).await
     } else if sd_args.type_of_inference == "model" {
-        process_job_sd(&args).await
+        process_job_sd_upload(&args).await
     } else {
         Err(ProcessSingleJobError::Other(anyhow!("inference type doesn't exist!")))
     }
 }
 
-pub async fn process_job_sd(
-    args: &StableDiffusionProcessArgs<'_>
-) -> Result<JobSuccessResult, ProcessSingleJobError> {
-
-    let job = args.job;
-    let deps = args.job_dependencies;
-    let mysql_pool = &deps.db.mysql_pool;
-
-    let sd_args = sd_args_from_job(&args).await?;
-
-    let sd_deps = match
-        &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
-    {
-        None => {
-            return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
-        }
-        Some(val) => { val }
-    };
-
-    let creator_ip_address = &job.creator_ip_address;
-    let creator_user_token: UserToken;
-
-    match &job.maybe_creator_user_token {
-        Some(token) => {
-            creator_user_token = UserToken::new_from_str(token);
-        }
-        None => {
-            return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
-        }
-    }
-
-    // The parameters will be updated on another screen perhaps?
-    // so right now it will fill with the availible  values.
-
-    let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
-    let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
-        .new_tempdir(&work_temp_dir)
-        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
-
-    let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
-    //let vae_path = work_temp_dir.path().join("vae.safetensors");
-    let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
-    let output_path = work_temp_dir.path().join("output");
-
-    info!("Paths to download to:");
-    info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
-    info!("vae_path: {:?}", vae_path);
-    info!("output_path: {:?}", output_path);
-
-    let download_url = match sd_args.maybe_upload_path {
-        Some(val) => { val }
-        None => { "".to_string() }
-    };
-
-    if download_url.len() == 0 {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
-        );
-    }
-
-    let file_name = "sd_checkpoint.safetensors";
-    let download_script = easyenv::get_env_string_or_default(
-        "DOWNLOAD_SCRIPT",
-        "download_internet_file.py"
-    );
-
-    let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
-        None, None, None);
-
-    info!("Downloading {}", download_url);
-
-    let download_filename = match
-        google_drive_downloader.download_file_with_file_name(
-            &download_url,
-            &work_temp_dir,
-            file_name
-        ).await
-    {
-        Ok(filename) => filename,
-        Err(_e) => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download"))),
-    };
-
-    let download_file_path = work_temp_dir.path().join(download_filename);
-
-    if file_exists(download_file_path.as_path()) == false {
-        return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download SD Model from Google")));
-    }
-
-    info!("File Retrieved at {}", download_file_path.display());
-
-    // download vae for model
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
-        .await
-        .map_err(|err| {
-            error!("could not download VAE: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
-        })?;
-
-    info!("VAE downloaded to: {:?}", &vae_path);
-
-//    // use this vae doesn't matter though
-//    // VAE token for now
-//    let vae_token = String::from("REPLACE_ME");
-//    let model_weight_vae = ModelWeightToken(vae_token);
-//    let vae_weight_record = get_weight_by_token(
-//        &model_weight_vae,
-//        false,
-//        &deps.db.mysql_pool
-//    ).await?;
-//    let vae_weight_record = match vae_weight_record {
-//        Some(val) => val,
-//        None => {
-//            return Err(
-//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-//            );
-//        }
-//    };
-//
-//    let vae_details = RemoteCloudBucketDetails::new(
-//        vae_weight_record.public_bucket_hash.clone(),
-//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-//    );
-//
-//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
-
-    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-
-    let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
-    let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
-
-    // run inference on loRA downloaded
-    let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
-        work_dir: work_temp_dir.path().to_path_buf(),
-        output_file: output_path.clone(),
-        stderr_output_file: &stderr_output_file,
-        stdout_output_file: &stdout_output_file,
-        prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
-        negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
-        number_of_samples:20,
-        samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        lora_path: PathBuf::from(""),
-        checkpoint_path: sd_checkpoint_path.clone(),
-        vae: vae_path.clone(),
-        batch_count: sd_args.maybe_batch_count.unwrap_or(1),
-    });
-
-    if !exit_status.is_success() {
-        error!("SD inference failed: {:?}", exit_status);
-
-        let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            warn!("Captured stderr output: {}", contents);
-
-            //match categorize_error(&contents)  {
-            //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //    warn!("Face not detected in source image");
-            //    error = ProcessSingleJobError::FaceDetectionFailure;
-            //  }
-            //  _ => {}
-            //}
-        }
-        return Err(error);
-    }
-
-
-    // check if file exists if it does not error out....
-    let path = output_path.clone();
-    let file_path = format!("{}_{}.png", path_to_string(path), 0);
-
-    if !file_exists(file_path.path()) {
-        error!("Output file did not exist: {:?}", file_path);
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            error!("Captured stderr output: {}", contents);
-        }
-
-        return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model")));
-    }
-
-    // If it worked and didn't fail! then we should save and create the weight.
-    // upload and create weights for loRA...
-    let weights_sd_descriptor = Box::new(WeightsSD15Descriptor{});
-    let metadata = remote_cloud_file_client.upload_file(weights_sd_descriptor,sd_checkpoint_path.to_str().unwrap_or_default()).await?;
-    // chekc the model hash for duplicated models.
-    let bucket_details = match metadata.bucket_details {
-        Some(metadata) => metadata, 
-        None => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
-        }
-    };
-
-    let model_weight_token = &ModelWeightToken::generate();
-    let model_weight_token_result = create_weight(CreateModelWeightsArgs {
-        token: &model_weight_token,
-        weights_type: WeightsType::StableDiffusion15,
-        weights_category: WeightsCategory::ImageGeneration,
-        title: sd_args.maybe_name.unwrap_or(String::from("")),
-        maybe_description_markdown: sd_args.maybe_description,
-        maybe_description_rendered_html: None,
-        creator_user_token: Some(&creator_user_token),
-        creator_ip_address,
-        creator_set_visibility: Default::default(),
-        maybe_last_update_user_token: None,
-        original_download_url: Some(download_url),
-        original_filename: None,
-        // file_size_bytes: metadata.file_size_bytes, // TODO(bt,2024-02-03): We need to migrate the column to be BIGINT
-        file_size_bytes: 0,
-        file_checksum_sha2: metadata.sha256_checksum,
-        public_bucket_hash: bucket_details.object_hash,
-        maybe_public_bucket_prefix: Some(bucket_details.prefix),
-        maybe_public_bucket_extension:Some(bucket_details.suffix),
-        version: sd_args.maybe_version.unwrap_or(0),
-        mysql_pool,
-    }).await?;
-
-    Ok(JobSuccessResult {
-        maybe_result_entity: Some(ResultEntity {
-            entity_type: InferenceResultType::UploadModel,
-            entity_token: model_weight_token_result.to_string(),
-        }),
-        inference_duration: Duration::from_secs(0),
-    })
-}
-
-pub async fn process_job_lora(
-    args: &StableDiffusionProcessArgs<'_>
-) -> Result<JobSuccessResult, ProcessSingleJobError> {
-
-    let job = args.job;
-    let deps = args.job_dependencies;
-    let mysql_pool = &deps.db.mysql_pool;
-
-    let sd_args = sd_args_from_job(&args).await?;
-
-    let sd_deps = match
-        &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
-    {
-        Some(val) => val,
-        None => return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies"))),
-    };
-
-    let creator_ip_address = &job.creator_ip_address;
-    let creator_user_token = match job.maybe_creator_user_token.as_deref() {
-        Some(token) => UserToken::new_from_str(token),
-        None => return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))),
-    };
-
-
-    let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
-    let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
-        .new_tempdir(&work_temp_dir)
-        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
-
-    let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
-    let lora_path = work_temp_dir.path().join("lora.safetensors"); // input path into execution
-    //let vae_path = work_temp_dir.path().join("vae.safetensors");
-    let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
-    let output_path = work_temp_dir.path().join("output");
-
-    info!("Paths to download to:");
-    info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
-    info!("lora_path: {:?}", lora_path);
-    info!("vae_path: {:?}", vae_path);
-    info!("output_path: {:?}", output_path);
-
-    let download_url = match sd_args.maybe_lora_upload_path {
-        Some(val) => { val }
-        None => { "".to_string() }
-    };
-
-    if download_url.len() == 0 {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
-        );
-    }
-
-    let file_name = "lora.safetensors";
-
-    let download_script = easyenv::get_env_string_or_default(
-        "DOWNLOAD_SCRIPT",
-        "download_internet_file.py"
-    );
-
-    // Download 
-    let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
-        None,
-        None, 
-        None);
-
-    info!("Downloading {}", download_url);
-
-    let download_filename = match
-        google_drive_downloader.download_file_with_file_name(
-            &download_url,
-            &work_temp_dir,
-            file_name
-        ).await
-    {
-        Ok(filename) => filename,
-        Err(_e) => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download")));
-        }
-    };
-
-    let download_file_path = work_temp_dir.path().join(download_filename);
-    info!("File Retrieved at {}", download_file_path.display());
-    if file_exists(download_file_path.as_path()) == false {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download loRA Model from Google Drive"))
-        );
-    }
-
-
-    // download model + vae for lora
-    // use lora with whatever checkpoint make it fixed id if it works submit it
-    let weight_token_sd_model_token = ModelWeightToken::new_from_str(&sd_deps.predefined_sd_weight_token);
-
-    info!("Using predefined SD weight token: {:?}", &weight_token_sd_model_token);
-
-    let sd_weight_record = get_weight_by_token(
-        &weight_token_sd_model_token,
-        false,
-        &deps.db.mysql_pool
-    ).await?;
-    let sd_weight_record = match sd_weight_record {
-        Some(val) => val,
-        None => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(anyhow!("No SD weight baked in for loRA upload inference check? thats a problem."))
-            );
-        }
-    };
-
-    let sd_weight_details = RemoteCloudBucketDetails::new(
-        sd_weight_record.public_bucket_hash.clone(),
-        sd_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-        sd_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-    );
-
-    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-    remote_cloud_file_client.download_file(sd_weight_details, path_to_string(sd_checkpoint_path.clone())).await?;
-
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
-        .await
-        .map_err(|err| {
-            error!("could not download VAE: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
-        })?;
-
-//    // use this vae doesn't matter though
-//    // VAE token for now
-//    let vae_token = String::from("REPLACE_ME");
-//    let model_weight_vae = ModelWeightToken(vae_token);
-//    let vae_weight_record = get_weight_by_token(
-//        &model_weight_vae,
-//        false,
-//        &deps.db.mysql_pool
-//    ).await?;
-//    let vae_weight_record = match vae_weight_record {
-//        Some(val) => val,
-//        None => {
-//            return Err(
-//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-//            );
-//        }
-//    };
-//
-//    let vae_details = RemoteCloudBucketDetails::new(
-//        vae_weight_record.public_bucket_hash.clone(),
-//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-//    );
-//
-//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
-
-    let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
-    let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
-
-    // run inference on loRA downloaded
-    let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
-        work_dir: work_temp_dir.path().to_path_buf(),
-        output_file: output_path.clone(),
-        stderr_output_file: &stderr_output_file,
-        stdout_output_file: &stdout_output_file,
-        prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
-        negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
-        number_of_samples:20,
-        samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        lora_path: lora_path.clone(),
-        checkpoint_path: sd_checkpoint_path.clone(),
-        vae: vae_path.clone(),
-        batch_count: sd_args.maybe_batch_count.unwrap_or(1),
-    });
-
-    if !exit_status.is_success() {
-        error!("SD inference failed: {:?}", exit_status);
-
-        let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            warn!("Captured stderr output: {}", contents);
-
-            //match categorize_error(&contents)  {
-            //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //    warn!("Face not detected in source image");
-            //    error = ProcessSingleJobError::FaceDetectionFailure;
-            //  }
-            //  _ => {}
-            //}
-        }
-        return Err(error);
-    }
-
-    // check if file exists if it does not error out....
-    let path = output_path.clone();
-    let file_path = format!("{}_{}.png", path_to_string(path), 0);
-    
-    if file_exists(file_path.path()) == false {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a LoRA"))
-        );
-    }
-
-    // If it worked and didn't fail! then we should save and create the weight.
-    // upload and create weights for loRA...
-    let lora_descriptor = Box::new(WeightsLoRADescriptor{});
-    let metadata = remote_cloud_file_client.upload_file(lora_descriptor,lora_path.to_str().unwrap_or_default()).await?;
-
-    let bucket_details = match metadata.bucket_details {
-        Some(metadata) => metadata, 
-        None => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
-        }
-    };
-
-    let model_weight_token = &ModelWeightToken::generate();
-    let model_weight_token_result = create_weight(CreateModelWeightsArgs {
-        token: &model_weight_token,
-        weights_type: WeightsType::LoRA,
-        weights_category: WeightsCategory::ImageGeneration,
-        title: sd_args.maybe_name.unwrap_or(String::from("")),
-        maybe_description_markdown: sd_args.maybe_description,
-        maybe_description_rendered_html: None,
-        creator_user_token: Some(&creator_user_token),
-        creator_ip_address,
-        creator_set_visibility: Default::default(),
-        maybe_last_update_user_token: None,
-        original_download_url: Some(download_url),
-        original_filename: None,
-        file_size_bytes: metadata.file_size_bytes,
-        file_checksum_sha2: metadata.sha256_checksum,
-        public_bucket_hash: bucket_details.object_hash,
-        maybe_public_bucket_prefix: Some(bucket_details.prefix),
-        maybe_public_bucket_extension:Some(bucket_details.suffix),
-        version: sd_args.maybe_version.unwrap_or(0),
-        mysql_pool,
-    }).await?;
-
-    Ok(JobSuccessResult {
-        maybe_result_entity: Some(ResultEntity {
-            entity_type: InferenceResultType::UploadModel,
-            entity_token: model_weight_token_result.to_string(),
-        }),
-        inference_duration: Duration::from_secs(0),
-    })
-
-}
-
-
-pub async fn process_job_inference(
-    args: &StableDiffusionProcessArgs<'_>
-) -> Result<JobSuccessResult, ProcessSingleJobError> {
-    let job = args.job;
-    let deps = args.job_dependencies;
-    let mysql_pool = &deps.db.mysql_pool;
-
-    let sd_args = sd_args_from_job(&args).await?;
-    let sd_deps: &crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies = match
-        &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
-    {
-        None => {
-            return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
-        }
-        Some(val) => { val }
-    };
-
-    let _creator_user_token: UserToken;
-    let _anon_user_token: Option<AnonymousVisitorTrackingToken>;
-
-    match &job.maybe_creator_anonymous_visitor_token {
-        Some(token) => {
-            let anonymous_visitor_token = AnonymousVisitorTrackingToken::new_from_str(token);
-            _anon_user_token = Some(anonymous_visitor_token);
-        }
-        None => {
-            _anon_user_token = None;
-        }
-    }
-
-    match &job.maybe_creator_user_token {
-        Some(token) => {
-            _creator_user_token = UserToken::new_from_str(token);
-        }
-        None => {
-            return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
-        }
-    }
-
-    let _job_progress_reporter = args.job_dependencies.clients.job_progress_reporter
-        .new_generic_inference(job.inference_job_token.as_str())
-        .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
-
-    //==================== TEMP DIR ==================== //
-
-    let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
-
-    //NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
-    let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
-        .new_tempdir(&work_temp_dir)
-        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
-
-    let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
-    let mut lora_path = work_temp_dir.path().join("lora.safetensors");
-    //let vae_path = work_temp_dir.path().join("vae.safetensors");
-    let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
-    let output_path = work_temp_dir.path().join("output");
-
-    info!("Paths to download to:");
-    info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
-    info!("lora_path: {:?}", lora_path);
-    info!("vae_path: {:?}", vae_path);
-    info!("output_path: {:?}", output_path);
-
-    // thread::sleep(seconds) to check the directory
-
-    // // Unpack loRA and Checkpoint
-    // // run inference by downloading from google drive.
-    let lora_token = sd_args.maybe_lora_model_token;
-    let weight_token = sd_args.maybe_sd_model_token.clone();
-
-    let retrieved_sd_record = match weight_token {
-        Some(ref token) => {
-            let retrieved_sd_record = get_weight_by_token(
-                &token,
-                false,
-                &deps.db.mysql_pool
-            ).await?;
-            match retrieved_sd_record {
-                Some(record) => record,
-                None => {
-                    return Err(
-                        ProcessSingleJobError::from_anyhow_error(anyhow!("no record of model!"))
-                    );
-                }
-            }
-        }
-        None => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))
-            );
-        }
-    };
-    // origin file name needs to be just the file name  /tmp/downloads_long_lived/temp_stable_diffusion_inference_32.8qJJljxWWZeD/output_0.png
-    // ignore if no lora token
-    let retrieved_loRA_record = match lora_token {
-        Some(token) => {
-            let retrieved_loRA_record = get_weight_by_token(
-                &token,
-                false,
-                &deps.db.mysql_pool
-            ).await?;
-
-            Some(retrieved_loRA_record)
-        }
-        None => None,
-    };
-
-    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-    
-    // Details for SD checkpoint
-    let details = RemoteCloudBucketDetails::new(
-        retrieved_sd_record.public_bucket_hash.clone(),
-        retrieved_sd_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-        retrieved_sd_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-    );
-
-    remote_cloud_file_client.download_file(
-        details,
-        path_to_string(sd_checkpoint_path.clone())
-    ).await?;
-
-    let mut lora_name = String::from("");
-    let mut lora_token = String::from("");
-
-    match retrieved_loRA_record {
-        Some(record) => {
-            match record {
-                Some(model_weight_record) => {
-                    let lora_details = RemoteCloudBucketDetails::new(
-                        model_weight_record.public_bucket_hash.clone(),
-                        model_weight_record.maybe_public_bucket_prefix
-                            .clone()
-                            .unwrap_or_else(|| "".to_string()),
-                        model_weight_record.maybe_public_bucket_extension
-                            .clone()
-                            .unwrap_or_else(|| "".to_string())
-                    );
-
-                    lora_name = model_weight_record.title;
-                    lora_token = model_weight_record.token.to_string();
-                    remote_cloud_file_client.download_file(
-                        lora_details,
-                        path_to_string(lora_path.clone())
-                    ).await?;
-                }
-                None => {
-                    lora_path.clear();
-                }
-            }
-        }
-        None => {
-            lora_path.clear();
-        }
-    }
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
-        .await
-        .map_err(|err| {
-            error!("could not download VAE: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
-        })?;
-
-//    // VAE token for now
-//    let vae_token = String::from("REPLACE_ME");
-//    let model_weight_vae = ModelWeightToken(vae_token);
-//
-//    let vae_weight_record = get_weight_by_token(
-//        &model_weight_vae,
-//        false,
-//        &deps.db.mysql_pool
-//    ).await?;
-//
-//    let vae_weight_record = match vae_weight_record {
-//        Some(val) => val,
-//        None => {
-//            return Err(
-//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-//            );
-//        }
-//    };
-//
-//    let vae_details = RemoteCloudBucketDetails::new(
-//        vae_weight_record.public_bucket_hash.clone(),
-//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-//    );
-//
-//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
-
-    let prompt = match sd_args.maybe_prompt {
-        Some(val) => val,
-        None => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No Prompt provided!")));
-        }
-    };
-
-    let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
-    let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
-
-    let number_of_samples = match sd_args.maybe_number_of_samples {
-        Some(val) => val,
-        None => 20,
-    };
-
-    let inference_start_time = Instant::now();
-
-    let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
-        work_dir: work_temp_dir.path().to_path_buf(),
-        output_file: output_path.clone(),
-        stderr_output_file: &stderr_output_file,
-        stdout_output_file: &stdout_output_file,
-        prompt: prompt.clone(),
-        negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
-        number_of_samples,
-        samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        lora_path: lora_path.clone(),
-        checkpoint_path: sd_checkpoint_path.clone(),
-        vae: vae_path.clone(),
-        batch_count: sd_args.maybe_batch_count.unwrap_or(1),
-    });
-
-    if !exit_status.is_success() {
-        error!("SD inference failed: {:?}", exit_status);
-
-        let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            warn!("Captured stderr output: {}", contents);
-
-            //match categorize_error(&contents)  {
-            //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //    warn!("Face not detected in source image");
-            //    error = ProcessSingleJobError::FaceDetectionFailure;
-            //  }
-            //  _ => {}
-            //}
-        }
-        return Err(error);
-    }
-
-
-    // hack to check the directory before clean up.
-    //   let thirtyMinutes = 1800;
-    //   thread::sleep(Duration::from_secs(thirtyMinutes));
-      // upload media and create a record.
-
-    let inference_duration = Instant::now().duration_since(inference_start_time);
-
-    // run a for loop for output images output_0 in the folder then use upload media.
-    // pngs
-
-    let mut entries = vec![];
-
-    let inputs = InferenceValues {
-        prompt: prompt.clone(),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        negative_prompt: sd_args.maybe_n_prompt,
-        lora_model_weight_token: Some(lora_token),
-        lora_name: Some(lora_name),
-        sampler: sd_args.maybe_sampler.unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        number_of_samples,
-    };
-
-    let inputs = match serde_json::to_string(&inputs) {
-        Ok(result) => result,
-        Err(_err) => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(anyhow!("couldn't serialize metadata."))
-            );
-        }
-    };
-
-
-    for i in 0..sd_args.maybe_batch_count.unwrap_or(1) {
-        let path = output_path.clone();
-
-        let file_path = format!("{}_{}.png", path_to_string(path), i);
-
-        println!("Upload File Path:{}", file_path);
-
-        let metadata = remote_cloud_file_client.upload_file(
-            Box::new(MediaImagePngDescriptor {}),
-            file_path.as_ref()
-        ).await?;
-
-        let bucket_details = match metadata.bucket_details {
-            Some(val) => { val }
-            None => {
-                return Err(
-                    ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-                );
-            }
-        };
-
-
-        // extra_file_modification_info: todo!(), // JSON ENCODED STRUCT
-        let media_file_token = insert_media_file_generic(InsertArgs {
-            pool: mysql_pool,
-            job,
-            media_type: MediaFileType::Image,
-            origin_category: MediaFileOriginCategory::Upload,
-            origin_product_category: MediaFileOriginProductCategory::ImageGeneration,
-            maybe_origin_model_type: Some(MediaFileOriginModelType::StableDiffusion15),
-            maybe_origin_model_token: weight_token.clone(),
-            maybe_origin_filename: Some(file_path),
-            is_batch_generated: true,
-            maybe_mime_type: Some(metadata.mimetype.as_ref()),
-            file_size_bytes: metadata.file_size_bytes,
-            maybe_duration_millis: Some(inference_duration.as_millis() as u64),
-            maybe_audio_encoding: None,
-            maybe_video_encoding: None,
-            maybe_frame_width: Some(sd_args.maybe_width.unwrap_or(512)),
-            maybe_frame_height: Some(sd_args.maybe_height.unwrap_or(512)),
-            checksum_sha2: metadata.sha256_checksum.as_str(),
-            public_bucket_directory_hash: bucket_details.object_hash.as_str(),
-            //maybe_public_bucket_prefix: Some(bucket_details.prefix.as_str()),
-            maybe_public_bucket_prefix: Some(SD_IMAGE_PREFIX),
-            //maybe_public_bucket_extension: Some(bucket_details.suffix.as_str()),
-            maybe_public_bucket_extension: Some(SD_IMAGE_SUFFIX),
-            extra_file_modification_info: Some(&inputs),
-            maybe_creator_file_synthetic_id_category: IdCategory::MediaFile,
-            maybe_creator_category_synthetic_id_category: IdCategory::ModelWeights,
-            maybe_mod_user_token: None,
-            is_generated_on_prem: args.job_dependencies.job.info.container.is_on_prem,
-            generated_by_worker: Some(&args.job_dependencies.job.info.container.hostname),
-            generated_by_cluster: Some(&args.job_dependencies.job.info.container.cluster_name),
-        }).await?;
-
-        let batch_generation_entity: BatchGenerationEntity = BatchGenerationEntity::MediaFile(
-            media_file_token.0
-        );
-        entries.push(batch_generation_entity);
-    }
-
-    let mut transaction = mysql_pool.begin().await.unwrap();
-    
-    let batch_token_result = insert_batch_generation_records(InsertBatchArgs {
-        entries,
-        transaction: &mut transaction,
-    }).await;
-
-    let batch_token = match batch_token_result {
-        Ok(v) => { v.to_string() }
-        Err(_err) => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(
-                    anyhow!("No batch token? something has failed.")
-                )
-            );
-        }
-    };
-
-    Ok(JobSuccessResult { // TODO: batch token needs to go here
-        maybe_result_entity: Some(ResultEntity {
-            entity_type: InferenceResultType::MediaFile,
-            entity_token: batch_token,
-        }),
-        inference_duration,
-    })
-}
 
 #[ignore]
 #[cfg(test)]

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_inference.rs

@@ -0,0 +1,414 @@
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use actix_web::dev::ResourcePath;
+use anyhow::anyhow;
+use log::{error, info, warn};
+use serde_json;
+
+use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
+use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
+use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
+use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use filesys::file_exists::file_exists;
+use filesys::path_to_string::path_to_string;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::{insert_batch_generation_records, InsertBatchArgs};
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::{insert_media_file_generic, InsertArgs};
+use mysql_queries::queries::model_weights::create::create_weight::{
+  create_weight,
+  CreateModelWeightsArgs,
+};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::image_generation::sd::process_job::{InferenceValues, sd_args_from_job, StableDiffusionProcessArgs};
+use crate::job::job_types::image_generation::sd::process_job_lora_upload::process_job_lora_upload;
+use crate::job::job_types::image_generation::sd::process_job_sd_upload::process_job_sd_upload;
+use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job_dependencies::JobDependencies;
+
+pub async fn process_job_inference(
+  args: &StableDiffusionProcessArgs<'_>
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+  let job = args.job;
+  let deps = args.job_dependencies;
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let sd_args = sd_args_from_job(&args).await?;
+  let sd_deps: &crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies = match
+  &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
+  {
+    None => {
+      return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
+    }
+    Some(val) => { val }
+  };
+
+  let _creator_user_token: UserToken;
+  let _anon_user_token: Option<AnonymousVisitorTrackingToken>;
+
+  match &job.maybe_creator_anonymous_visitor_token {
+    Some(token) => {
+      let anonymous_visitor_token = AnonymousVisitorTrackingToken::new_from_str(token);
+      _anon_user_token = Some(anonymous_visitor_token);
+    }
+    None => {
+      _anon_user_token = None;
+    }
+  }
+
+  match &job.maybe_creator_user_token {
+    Some(token) => {
+      _creator_user_token = UserToken::new_from_str(token);
+    }
+    None => {
+      return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
+    }
+  }
+
+  let _job_progress_reporter = args.job_dependencies.clients.job_progress_reporter
+      .new_generic_inference(job.inference_job_token.as_str())
+      .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
+
+  //==================== TEMP DIR ==================== //
+
+  let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
+
+  //NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
+  let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  //let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
+  let mut lora_path = work_temp_dir.path().join("lora.safetensors");
+  //let vae_path = work_temp_dir.path().join("vae.safetensors");
+  let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
+  let output_path = work_temp_dir.path().join("output");
+
+  info!("Paths to download to:");
+  info!("lora_path: {:?}", lora_path);
+  info!("vae_path: {:?}", vae_path);
+  info!("output_path: {:?}", output_path);
+
+  // thread::sleep(seconds) to check the directory
+
+  // // Unpack loRA and Checkpoint
+  // // run inference by downloading from google drive.
+  let lora_token = sd_args.maybe_lora_model_token;
+///  let weight_token = sd_args.maybe_sd_model_token.clone();
+
+  let sd_model_weight_token = match sd_args.maybe_sd_model_token {
+    None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))),
+    Some(ref token) => token,
+  };
+
+  let sd_model_weight = get_weight_by_token(
+    sd_model_weight_token,
+    false,
+    &deps.db.mysql_pool
+  ).await?;
+
+  let sd_model_weight = match sd_model_weight {
+    None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no record of sd model!"))),
+    Some(record) => record,
+  };
+
+  let sd_checkpoint_path = args.job_dependencies
+      .fs
+      .model_weights_cache_directory
+      .get_model_weight_from_cache_or_bucket(&sd_model_weight)
+      .await?;
+
+  info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
+
+//  let retrieved_sd_record = match sd_args.maybe_sd_model_token {
+//    None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))),
+//    Some(ref token) => {
+//      let retrieved_sd_record = get_weight_by_token(
+//        &token,
+//        false,
+//        &deps.db.mysql_pool
+//      ).await?;
+//      match retrieved_sd_record {
+//        Some(record) => record,
+//        None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no record of model!"))),
+//      }
+//    }
+//  };
+
+  // origin file name needs to be just the file name  /tmp/downloads_long_lived/temp_stable_diffusion_inference_32.8qJJljxWWZeD/output_0.png
+  // ignore if no lora token
+
+  let mut maybe_lora_record = None;
+
+  if let Some(token) = lora_token {
+    maybe_lora_record = get_weight_by_token(
+      &token,
+      false,
+      &deps.db.mysql_pool
+    ).await?;
+  }
+
+  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+
+///  // Details for SD checkpoint
+///  let details = RemoteCloudBucketDetails::new(
+///    retrieved_sd_record.public_bucket_hash.clone(),
+///    retrieved_sd_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+///    retrieved_sd_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+///  );
+///
+///  remote_cloud_file_client.download_file(
+///    details,
+///    path_to_string(sd_checkpoint_path.clone())
+///  ).await?;
+
+  let mut lora_name = String::from("");
+  let mut lora_token = String::from("");
+
+  match maybe_lora_record {
+    None => {
+      lora_path.clear();
+    }
+    Some(lora_record) => {
+/////      let lora_details = RemoteCloudBucketDetails::new(
+/////        lora_record.public_bucket_hash.clone(),
+/////        lora_record.maybe_public_bucket_prefix
+/////            .clone()
+/////            .unwrap_or_else(|| "".to_string()),
+/////        lora_record.maybe_public_bucket_extension
+/////            .clone()
+/////            .unwrap_or_else(|| "".to_string())
+/////      );
+/////
+/////
+/////      remote_cloud_file_client.download_file(
+/////        lora_details,
+/////        path_to_string(lora_path.clone())
+/////      ).await?;
+
+      lora_name = lora_record.title.to_string();
+      lora_token = lora_record.token.to_string();
+
+      lora_path = args.job_dependencies
+          .fs
+          .model_weights_cache_directory
+          .get_model_weight_from_cache_or_bucket(&lora_record)
+          .await?;
+    }
+  }
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
+      .await
+      .map_err(|err| {
+        error!("could not download VAE: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
+      })?;
+
+  let prompt = match sd_args.maybe_prompt {
+    Some(val) => val,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No Prompt provided!")));
+    }
+  };
+
+  let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
+  let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
+
+  let number_of_samples = match sd_args.maybe_number_of_samples {
+    Some(val) => val,
+    None => 20,
+  };
+
+  let inference_start_time = Instant::now();
+
+  let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
+    work_dir: work_temp_dir.path().to_path_buf(),
+    output_file: output_path.clone(),
+    stderr_output_file: &stderr_output_file,
+    stdout_output_file: &stdout_output_file,
+    prompt: prompt.clone(),
+    negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
+    number_of_samples,
+    samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    lora_path: lora_path.clone(),
+    checkpoint_path: sd_checkpoint_path.clone(),
+    vae: vae_path.clone(),
+    batch_count: sd_args.maybe_batch_count.unwrap_or(1),
+  });
+
+  if !exit_status.is_success() {
+    error!("SD inference failed: {:?}", exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+    return Err(error);
+  }
+
+
+  // hack to check the directory before clean up.
+  //   let thirtyMinutes = 1800;
+  //   thread::sleep(Duration::from_secs(thirtyMinutes));
+  // upload media and create a record.
+
+  let inference_duration = Instant::now().duration_since(inference_start_time);
+
+  // run a for loop for output images output_0 in the folder then use upload media.
+  // pngs
+
+  let mut entries = vec![];
+
+  let inputs = InferenceValues {
+    prompt: prompt.clone(),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    negative_prompt: sd_args.maybe_n_prompt,
+    lora_model_weight_token: Some(lora_token),
+    lora_name: Some(lora_name),
+    sampler: sd_args.maybe_sampler.unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    number_of_samples,
+  };
+
+  let inputs = match serde_json::to_string(&inputs) {
+    Ok(result) => result,
+    Err(_err) => {
+      return Err(
+        ProcessSingleJobError::from_anyhow_error(anyhow!("couldn't serialize metadata."))
+      );
+    }
+  };
+
+  let mut maybe_first_media_file_token = None;
+
+  for i in 0..sd_args.maybe_batch_count.unwrap_or(1) {
+    let path = output_path.clone();
+
+    let file_path = format!("{}_{}.png", path_to_string(path), i);
+
+    println!("Upload File Path:{}", file_path);
+
+    let metadata = remote_cloud_file_client.upload_file(
+      Box::new(MediaImagePngDescriptor {}),
+      file_path.as_ref()
+    ).await?;
+
+    let bucket_details = match metadata.bucket_details {
+      Some(val) => { val }
+      None => {
+        return Err(
+          ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
+        );
+      }
+    };
+
+
+    // extra_file_modification_info: todo!(), // JSON ENCODED STRUCT
+    let (media_file_token, _id) = insert_media_file_generic(InsertArgs {
+      pool: mysql_pool,
+      job,
+      media_type: MediaFileType::Image,
+      origin_category: MediaFileOriginCategory::Upload,
+      origin_product_category: MediaFileOriginProductCategory::ImageGeneration,
+      maybe_origin_model_type: Some(MediaFileOriginModelType::StableDiffusion15),
+      maybe_origin_model_token: Some(sd_model_weight_token.clone()),
+      maybe_origin_filename: Some(file_path),
+      is_batch_generated: true,
+      maybe_mime_type: Some(metadata.mimetype.as_ref()),
+      file_size_bytes: metadata.file_size_bytes,
+      maybe_duration_millis: Some(inference_duration.as_millis() as u64),
+      maybe_audio_encoding: None,
+      maybe_video_encoding: None,
+      maybe_frame_width: Some(sd_args.maybe_width.unwrap_or(512)),
+      maybe_frame_height: Some(sd_args.maybe_height.unwrap_or(512)),
+      checksum_sha2: metadata.sha256_checksum.as_str(),
+      public_bucket_directory_hash: bucket_details.object_hash.as_str(),
+      maybe_public_bucket_prefix: Some(bucket_details.prefix.as_str()),
+      maybe_public_bucket_extension: Some(bucket_details.suffix.as_str()),
+      extra_file_modification_info: Some(&inputs),
+      maybe_creator_file_synthetic_id_category: IdCategory::MediaFile,
+      maybe_creator_category_synthetic_id_category: IdCategory::ModelWeights,
+      maybe_mod_user_token: None,
+      is_generated_on_prem: args.job_dependencies.job.info.container.is_on_prem,
+      generated_by_worker: Some(&args.job_dependencies.job.info.container.hostname),
+      generated_by_cluster: Some(&args.job_dependencies.job.info.container.cluster_name),
+    }).await?;
+
+    if maybe_first_media_file_token.is_none() {
+      maybe_first_media_file_token = Some(media_file_token.clone());
+    }
+
+    let batch_generation_entity: BatchGenerationEntity = BatchGenerationEntity::MediaFile(
+      media_file_token
+    );
+
+    entries.push(batch_generation_entity);
+  }
+
+  let mut transaction = mysql_pool.begin().await.unwrap();
+
+  let batch_token_result = insert_batch_generation_records(InsertBatchArgs {
+    entries,
+    transaction: &mut transaction,
+  }).await;
+
+  let batch_token = match batch_token_result {
+    Ok(v) => { v.to_string() }
+    Err(_err) => {
+      return Err(
+        ProcessSingleJobError::from_anyhow_error(
+          anyhow!("No batch token? something has failed.")
+        )
+      );
+    }
+  };
+
+  // TODO(bt,2024-02-12): Return the batch token instead. (We're not ready for that.)
+  Ok(JobSuccessResult {
+    inference_duration,
+    maybe_result_entity: maybe_first_media_file_token
+        .map(|media_file_token| {
+          ResultEntity {
+            entity_type: InferenceResultType::MediaFile,
+            entity_token: media_file_token.to_string(),
+          }
+        }),
+  })
+}

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_lora_upload.rs

@@ -0,0 +1,318 @@
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use actix_web::dev::ResourcePath;
+use anyhow::anyhow;
+use log::{error, info, warn};
+use serde_json;
+
+use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
+use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
+use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
+use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use filesys::file_exists::file_exists;
+use filesys::path_to_string::path_to_string;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::insert_batch_generation_records;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::InsertBatchArgs;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::insert_media_file_generic;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::InsertArgs;
+use mysql_queries::queries::model_weights::create::create_weight::{
+  create_weight,
+  CreateModelWeightsArgs,
+};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::image_generation::sd::process_job::{sd_args_from_job, StableDiffusionProcessArgs};
+use crate::job::job_types::image_generation::sd::process_job_sd_upload::process_job_sd_upload;
+use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job_dependencies::JobDependencies;
+
+pub async fn process_job_lora_upload(
+  args: &StableDiffusionProcessArgs<'_>
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let job = args.job;
+  let deps = args.job_dependencies;
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let sd_args = sd_args_from_job(&args).await?;
+
+  let sd_deps = match
+  &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
+  {
+    Some(val) => val,
+    None => return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies"))),
+  };
+
+  let creator_ip_address = &job.creator_ip_address;
+  let creator_user_token = match job.maybe_creator_user_token.as_deref() {
+    Some(token) => UserToken::new_from_str(token),
+    None => return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))),
+  };
+
+
+  let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
+  let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
+  let lora_path = work_temp_dir.path().join("lora.safetensors"); // input path into execution
+  //let vae_path = work_temp_dir.path().join("vae.safetensors");
+  let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
+  let output_path = work_temp_dir.path().join("output");
+
+  info!("Paths to download to:");
+  info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
+  info!("lora_path: {:?}", lora_path);
+  info!("vae_path: {:?}", vae_path);
+  info!("output_path: {:?}", output_path);
+
+  let download_url = match sd_args.maybe_lora_upload_path {
+    Some(val) => { val }
+    None => { "".to_string() }
+  };
+
+  if download_url.len() == 0 {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
+    );
+  }
+
+  let file_name = "lora.safetensors";
+
+  let download_script = easyenv::get_env_string_or_default(
+    "DOWNLOAD_SCRIPT",
+    "download_internet_file.py"
+  );
+
+  // Download
+  let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
+                                                                None,
+                                                                None,
+                                                                None);
+
+  info!("Downloading {}", download_url);
+
+  let download_filename = match
+  google_drive_downloader.download_file_with_file_name(
+    &download_url,
+    &work_temp_dir,
+    file_name
+  ).await
+  {
+    Ok(filename) => filename,
+    Err(_e) => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download")));
+    }
+  };
+
+  let download_file_path = work_temp_dir.path().join(download_filename);
+  info!("File Retrieved at {}", download_file_path.display());
+  if file_exists(download_file_path.as_path()) == false {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download loRA Model from Google Drive"))
+    );
+  }
+
+
+//  // download model + vae for lora
+//  // use lora with whatever checkpoint make it fixed id if it works submit it
+//  let weight_token_sd_model_token = ModelWeightToken::new_from_str(&sd_deps.predefined_sd_weight_token);
+//
+//  info!("Using predefined SD weight token: {:?}", &weight_token_sd_model_token);
+//
+//  let sd_weight_record = get_weight_by_token(
+//    &weight_token_sd_model_token,
+//    false,
+//    &deps.db.mysql_pool
+//  ).await?;
+//  let sd_weight_record = match sd_weight_record {
+//    Some(val) => val,
+//    None => {
+//      return Err(
+//        ProcessSingleJobError::from_anyhow_error(anyhow!("No SD weight baked in for loRA upload inference check? thats a problem."))
+//      );
+//    }
+//  };
+//
+//  let sd_weight_details = RemoteCloudBucketDetails::new(
+//    sd_weight_record.public_bucket_hash.clone(),
+//    sd_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+//    sd_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+//  );
+//
+//  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+//  remote_cloud_file_client.download_file(sd_weight_details, path_to_string(sd_checkpoint_path.clone())).await?;
+
+  info!("Downloading predefined SD weight from: {:?} to {:?}",
+    &sd_deps.predefined_sd_weight_bucket_path,
+    &sd_checkpoint_path);
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.predefined_sd_weight_bucket_path, &sd_checkpoint_path)
+      .await
+      .map_err(|err| {
+        error!("could not download predefined SD weight: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download predefined SD weight: {:?}", err))
+      })?;
+
+  info!("Downloading predefined VAE weight from: {:?} to {:?}",
+    &sd_deps.vae_bucket_path,
+    &vae_path);
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
+      .await
+      .map_err(|err| {
+        error!("could not download VAE: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
+      })?;
+
+//    // use this vae doesn't matter though
+//    // VAE token for now
+//    let vae_token = String::from("REPLACE_ME");
+//    let model_weight_vae = ModelWeightToken(vae_token);
+//    let vae_weight_record = get_weight_by_token(
+//        &model_weight_vae,
+//        false,
+//        &deps.db.mysql_pool
+//    ).await?;
+//    let vae_weight_record = match vae_weight_record {
+//        Some(val) => val,
+//        None => {
+//            return Err(
+//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
+//            );
+//        }
+//    };
+//
+//    let vae_details = RemoteCloudBucketDetails::new(
+//        vae_weight_record.public_bucket_hash.clone(),
+//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+//    );
+//
+//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
+
+  let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
+  let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
+
+  // run inference on loRA downloaded
+  let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
+    work_dir: work_temp_dir.path().to_path_buf(),
+    output_file: output_path.clone(),
+    stderr_output_file: &stderr_output_file,
+    stdout_output_file: &stdout_output_file,
+    prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
+    negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
+    number_of_samples:20,
+    samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    lora_path: lora_path.clone(),
+    checkpoint_path: sd_checkpoint_path.clone(),
+    vae: vae_path.clone(),
+    batch_count: sd_args.maybe_batch_count.unwrap_or(1),
+  });
+
+  if !exit_status.is_success() {
+    error!("SD inference failed: {:?}", exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+    return Err(error);
+  }
+
+  // check if file exists if it does not error out....
+  let path = output_path.clone();
+  let file_path = format!("{}_{}.png", path_to_string(path), 0);
+
+  if file_exists(file_path.path()) == false {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a LoRA"))
+    );
+  }
+
+  // If it worked and didn't fail! then we should save and create the weight.
+  // upload and create weights for loRA...
+  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+  let lora_descriptor = Box::new(WeightsLoRADescriptor{});
+  let metadata = remote_cloud_file_client.upload_file(lora_descriptor,lora_path.to_str().unwrap_or_default()).await?;
+
+  let bucket_details = match metadata.bucket_details {
+    Some(metadata) => metadata,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
+    }
+  };
+
+  let model_weight_token = &ModelWeightToken::generate();
+  let model_weight_token_result = create_weight(CreateModelWeightsArgs {
+    token: &model_weight_token,
+    weights_type: WeightsType::LoRA,
+    weights_category: WeightsCategory::ImageGeneration,
+    title: sd_args.maybe_name.unwrap_or(String::from("")),
+    maybe_description_markdown: sd_args.maybe_description,
+    maybe_description_rendered_html: None,
+    creator_user_token: Some(&creator_user_token),
+    creator_ip_address,
+    creator_set_visibility: Default::default(),
+    maybe_last_update_user_token: None,
+    original_download_url: Some(download_url),
+    original_filename: None,
+    file_size_bytes: metadata.file_size_bytes,
+    file_checksum_sha2: metadata.sha256_checksum,
+    public_bucket_hash: bucket_details.object_hash,
+    maybe_public_bucket_prefix: Some(bucket_details.prefix),
+    maybe_public_bucket_extension:Some(bucket_details.suffix),
+    version: sd_args.maybe_version.unwrap_or(0),
+    mysql_pool,
+  }).await?;
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::UploadModel,
+      entity_token: model_weight_token_result.to_string(),
+    }),
+    inference_duration: Duration::from_secs(0),
+  })
+
+}

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_sd_upload.rs

@@ -0,0 +1,280 @@
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use actix_web::dev::ResourcePath;
+use anyhow::anyhow;
+use log::{error, info, warn};
+use serde_json;
+
+use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
+use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
+use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
+use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use filesys::file_exists::file_exists;
+use filesys::path_to_string::path_to_string;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::insert_batch_generation_records;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::InsertBatchArgs;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::insert_media_file_generic;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::InsertArgs;
+use mysql_queries::queries::model_weights::create::create_weight::{
+  create_weight,
+  CreateModelWeightsArgs,
+};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::image_generation::sd::process_job::{sd_args_from_job, StableDiffusionProcessArgs};
+use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job_dependencies::JobDependencies;
+
+pub async fn process_job_sd_upload(
+  args: &StableDiffusionProcessArgs<'_>
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let job = args.job;
+  let deps = args.job_dependencies;
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let sd_args = sd_args_from_job(&args).await?;
+
+  let sd_deps = match
+  &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
+  {
+    None => {
+      return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
+    }
+    Some(val) => { val }
+  };
+
+  let creator_ip_address = &job.creator_ip_address;
+  let creator_user_token: UserToken;
+
+  match &job.maybe_creator_user_token {
+    Some(token) => {
+      creator_user_token = UserToken::new_from_str(token);
+    }
+    None => {
+      return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
+    }
+  }
+
+  // The parameters will be updated on another screen perhaps?
+  // so right now it will fill with the availible  values.
+
+  let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
+  let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
+  //let vae_path = work_temp_dir.path().join("vae.safetensors");
+  let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
+  let output_path = work_temp_dir.path().join("output");
+
+  info!("Paths to download to:");
+  info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
+  info!("vae_path: {:?}", vae_path);
+  info!("output_path: {:?}", output_path);
+
+  let download_url = match sd_args.maybe_upload_path {
+    Some(val) => { val }
+    None => { "".to_string() }
+  };
+
+  if download_url.len() == 0 {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
+    );
+  }
+
+  let file_name = "sd_checkpoint.safetensors";
+  let download_script = easyenv::get_env_string_or_default(
+    "DOWNLOAD_SCRIPT",
+    "download_internet_file.py"
+  );
+
+  let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
+                                                                None, None, None);
+
+  info!("Downloading {}", download_url);
+
+  let download_filename = match
+  google_drive_downloader.download_file_with_file_name(
+    &download_url,
+    &work_temp_dir,
+    file_name
+  ).await
+  {
+    Ok(filename) => filename,
+    Err(_e) => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download"))),
+  };
+
+  let download_file_path = work_temp_dir.path().join(download_filename);
+
+  if file_exists(download_file_path.as_path()) == false {
+    return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download SD Model from Google")));
+  }
+
+  info!("File Retrieved at {}", download_file_path.display());
+
+  // download vae for model
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
+      .await
+      .map_err(|err| {
+        error!("could not download VAE: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
+      })?;
+
+  info!("VAE downloaded to: {:?}", &vae_path);
+
+//    // use this vae doesn't matter though
+//    // VAE token for now
+//    let vae_token = String::from("REPLACE_ME");
+//    let model_weight_vae = ModelWeightToken(vae_token);
+//    let vae_weight_record = get_weight_by_token(
+//        &model_weight_vae,
+//        false,
+//        &deps.db.mysql_pool
+//    ).await?;
+//    let vae_weight_record = match vae_weight_record {
+//        Some(val) => val,
+//        None => {
+//            return Err(
+//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
+//            );
+//        }
+//    };
+//
+//    let vae_details = RemoteCloudBucketDetails::new(
+//        vae_weight_record.public_bucket_hash.clone(),
+//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+//    );
+//
+//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
+
+  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+
+  let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
+  let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
+
+  // run inference on loRA downloaded
+  let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
+    work_dir: work_temp_dir.path().to_path_buf(),
+    output_file: output_path.clone(),
+    stderr_output_file: &stderr_output_file,
+    stdout_output_file: &stdout_output_file,
+    prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
+    negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
+    number_of_samples:20,
+    samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    lora_path: PathBuf::from(""),
+    checkpoint_path: sd_checkpoint_path.clone(),
+    vae: vae_path.clone(),
+    batch_count: sd_args.maybe_batch_count.unwrap_or(1),
+  });
+
+  if !exit_status.is_success() {
+    error!("SD inference failed: {:?}", exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+    return Err(error);
+  }
+
+  // check if file exists if it does not error out....
+  let path = output_path.clone();
+  let file_path = format!("{}_{}.png", path_to_string(path), 0);
+
+  if !file_exists(file_path.path()) {
+    error!("Output file did not exist: {:?}", file_path);
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      error!("Captured stderr output: {}", contents);
+    }
+
+    return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model")));
+  }
+
+  // If it worked and didn't fail! then we should save and create the weight.
+  // upload and create weights for loRA...
+  let weights_sd_descriptor = Box::new(WeightsSD15Descriptor{});
+  let metadata = remote_cloud_file_client.upload_file(weights_sd_descriptor,sd_checkpoint_path.to_str().unwrap_or_default()).await?;
+  // chekc the model hash for duplicated models.
+  let bucket_details = match metadata.bucket_details {
+    Some(metadata) => metadata,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
+    }
+  };
+
+  let model_weight_token: &ModelWeightToken = &ModelWeightToken::generate();
+  let model_weight_token_result = create_weight(CreateModelWeightsArgs {
+    token: &model_weight_token,
+    weights_type: WeightsType::StableDiffusion15,
+    weights_category: WeightsCategory::ImageGeneration,
+    title: sd_args.maybe_name.unwrap_or(String::from("")),
+    maybe_description_markdown: sd_args.maybe_description,
+    maybe_description_rendered_html: None,
+    creator_user_token: Some(&creator_user_token),
+    creator_ip_address,
+    creator_set_visibility: Default::default(),
+    maybe_last_update_user_token: None,
+    original_download_url: Some(download_url),
+    original_filename: None,
+    // file_size_bytes: metadata.file_size_bytes, // TODO(bt,2024-02-03): We need to migrate the column to be BIGINT
+    file_size_bytes: 0,
+    file_checksum_sha2: metadata.sha256_checksum,
+    public_bucket_hash: bucket_details.object_hash,
+    maybe_public_bucket_prefix: Some(bucket_details.prefix),
+    maybe_public_bucket_extension:Some(bucket_details.suffix),
+    version: sd_args.maybe_version.unwrap_or(0),
+    mysql_pool,
+  }).await?;
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::UploadModel,
+      entity_token: model_weight_token_result.to_string(),
+    }),
+    inference_duration: Duration::from_secs(0),
+  })
+}

crates/service/job/inference_job/src/job/job_types/image_generation/sd/stable_diffusion_dependencies.rs

@@ -9,6 +9,7 @@ pub struct StableDiffusionDependencies {
   pub inference_command: StableDiffusionInferenceCommand,
   pub vae_bucket_path: String,
   pub predefined_sd_weight_token: String,
+  pub predefined_sd_weight_bucket_path: String,
   pub predefined_lora_weight_token: String,
 }
 
@@ -19,6 +20,7 @@ impl StableDiffusionDependencies {
       inference_command: StableDiffusionInferenceCommand::from_env()?,
       vae_bucket_path: easyenv::get_env_string_required("SD_VAE_BUCKET_PATH")?,
       predefined_sd_weight_token: easyenv::get_env_string_required("SD_PREDEFINED_SD_WEIGHT_TOKEN")?,
+      predefined_sd_weight_bucket_path: easyenv::get_env_string_required("SD_PREDEFINED_SD_WEIGHT_BUCKET_PATH")?,
       predefined_lora_weight_token: easyenv::get_env_string_required("SD_PREDEFINED_LORA_WEIGHT_TOKEN")?,
     })
   }

crates/service/job/inference_job/src/job/job_types/mod.rs

@@ -6,3 +6,4 @@ pub mod image_generation;
 pub(crate) mod mocap;
 pub(crate) mod workflow;
 pub mod format_conversion;
+pub mod bevy_to_workflow;

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/comfy_ui_dependencies.rs

@@ -6,7 +6,6 @@ use crate::job::job_types::workflow::comfy_ui::comfy_ui_inference_command::Comfy
 pub struct ComfyDependencies {
     pub inference_command: ComfyInferenceCommand,
     pub dependency_tokens: RequiredModels,
-    pub workflow_bucket_path: String,
 }
 
 impl ComfyDependencies {
@@ -14,7 +13,6 @@ impl ComfyDependencies {
         Ok(Self {
             inference_command: ComfyInferenceCommand::from_env()?,
             dependency_tokens: RequiredModels::init(),
-            workflow_bucket_path: easyenv::get_env_string_required("COMFY_WORKFLOW_BUCKET_PATH")?,
         })
     }
 }
@@ -133,6 +131,14 @@ impl RequiredModels {
                     location: PathBuf::from("input/example.png"),
                     url: "https://storage.googleapis.com/vocodes-public/comfyui-deps/example.png".to_string(),
                 },
+                ComfyDependency {
+                    location: PathBuf::from("custom_nodes/ComfyUI-AnimateDiff-Evolved/models/sd15_t2v_beta_motion.ckpt"),
+                    url: "https://storage.googleapis.com/vocodes-public/comfyui-deps/sd15_t2v_beta_motion.ckpt".to_string(),
+                },
+                ComfyDependency {
+                    location: PathBuf::from("loras/LCM_LoRA_Weights_SD15.safetensors"),
+                    url: "https://storage.googleapis.com/vocodes-public/comfyui-deps/LCM_LoRA_Weights_SD15.safetensors".to_string(),
+                },
             ],
         }
     }

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/process_job.rs

@@ -136,7 +136,7 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
                 info!("Checking if path exists: {:?}", dep_path);
                 if !dep_path.exists() {
                     warn!("Path does not exist: {:?}", dep_path);
-                    download_file(model.url.clone(), dep_path.clone()).await.map_err(|e| ProcessSingleJobError::Other(e))?;
+                    download_file(model.url.clone(), dep_path.clone()).await.map_err(|e| ProcessSingleJobError::Other(e.into()))?;
                     info!("Downloaded model to {:?}", dep_path);
                 }
             }
@@ -155,31 +155,21 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
 
     let workflow_path = workflow_dir.join("prompt.json").to_str().unwrap().to_string();
 
-    //let retrieved_workflow_record =  get_weight_by_token(
-    //    job_args.workflow_source,
-    //    false,
-    //    &deps.db.mysql_pool
-    //).await?.unwrap();
+    let retrieved_workflow_record =  get_weight_by_token(
+       job_args.workflow_source,
+       false,
+       &deps.db.mysql_pool
+    ).await?.unwrap();
 
-    //let bucket_details = RemoteCloudBucketDetails {
-    //    object_hash: retrieved_workflow_record.public_bucket_hash,
-    //    prefix: retrieved_workflow_record.maybe_public_bucket_prefix.unwrap(),
-    //    suffix: retrieved_workflow_record.maybe_public_bucket_extension.unwrap(),
-    //};
-    //remote_cloud_file_client.download_file(bucket_details, workflow_path.clone()).await?;
+    let bucket_details = RemoteCloudBucketDetails {
+       object_hash: retrieved_workflow_record.public_bucket_hash,
+       prefix: retrieved_workflow_record.maybe_public_bucket_prefix.unwrap(),
+       suffix: retrieved_workflow_record.maybe_public_bucket_extension.unwrap(),
+    };
+    remote_cloud_file_client.download_file(bucket_details, workflow_path.clone()).await?;
+    info!("Downloaded workflow to {:?}", workflow_path);
 
-    info!("Downloading workflow {:?}", &comfy_deps.workflow_bucket_path);
-    info!("Downloading workflow to {:?}", workflow_path);
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&comfy_deps.workflow_bucket_path, &workflow_path)
-        .await
-        .map_err(|err| {
-            error!("could not download workflow: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download workflow: {:?}", err))
-        })?;
+    // download_file(comfy_deps.workflow_bucket_path.clone(), PathBuf::from(workflow_path.clone())).await.map_err(|e| ProcessSingleJobError::Other(e))?;
 
     let maybe_args = job.maybe_inference_args
         .as_ref()
@@ -203,14 +193,13 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
         let mut prompt_json: Value = serde_json::from_reader(prompt_file).unwrap();
         // Modify json
         for (path, new_value) in modifications {
-            prompt_json = replace_json_value(prompt_json, &path, new_value).map_err(|e| ProcessSingleJobError::Other(e))?;
+            prompt_json = replace_json_value(prompt_json, &path, new_value).map_err(|e| ProcessSingleJobError::Other(e.into()))?;
         }
         // Save prompt.json
         let prompt_file = File::create(&workflow_path).unwrap();
         serde_json::to_writer(prompt_file, &prompt_json).unwrap();
     }
 
-
     // ==================== QUERY AND DOWNLOAD FILES ==================== //
 
     // Download SD model if specified
@@ -475,6 +464,9 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
 
     // ==================== SAVE RECORDS ==================== //
 
+    // create a json detailing the args used to create the media
+    let args_json = serde_json::to_string(&job_args).map_err(|e| ProcessSingleJobError::Other(e.into()))?;
+
     info!("Saving ComfyUI result (media_files table record) ...");
 
     let (media_file_token, id) = insert_media_file_from_comfy_ui(InsertArgs {
@@ -490,6 +482,7 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result<JobSuccessResu
         worker_hostname: &args.job_dependencies.job.info.container.hostname,
         worker_cluster: &args.job_dependencies.job.info.container.cluster_name,
         media_file_type: media_type,
+        extra_file_modification_info: Some(&args_json),
     })
         .await
         .map_err(|e| ProcessSingleJobError::Other(e))?;

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/validate_job.rs

@@ -9,6 +9,7 @@ use tokens::tokens::model_weights::ModelWeightToken;
 
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 
+#[derive(Serialize, Debug)]
 pub struct JobArgs<'a> {
     pub workflow_source: &'a ModelWeightToken,
     pub output_path: &'a String,

crates/service/job/inference_job/src/job_dependencies.rs

@@ -24,9 +24,12 @@ use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmen
 use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
 
 use crate::job_specific_dependencies::JobSpecificDependencies;
+use crate::util::instrumentation::JobInstruments;
+use crate::util::model_weights_cache::model_weights_cache_directory::ModelWeightsCacheDirectory;
 use crate::util::scoped_execution::ScopedExecution;
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
 
+
 pub struct JobDependencies {
   /// Database dependencies.
   pub db: DatabaseDependencies,
@@ -42,6 +45,9 @@ pub struct JobDependencies {
 
   /// Job controls, stats, etc.
   pub job: JobSystemDependencies,
+
+  /// otel instruments
+  pub job_instruments: JobInstruments,
 }
 
 pub struct JobSystemDependencies {
@@ -159,6 +165,10 @@ pub struct FileSystemDetails {
   /// Directory to store long-term downloads (models)
   /// (In prod, typically model files from GCS / NFS PVC mount)
   pub semi_persistent_cache: SemiPersistentCacheDir,
+
+  /// The new cache scheme for model weights.
+  /// This should replace `semi_persistent_cache` for new users.
+  pub model_weights_cache_directory: ModelWeightsCacheDirectory,
 }
 
 // TODO: Move into the appropriate job-specific dependencies object.

crates/service/job/inference_job/src/job_specific_dependencies.rs

@@ -1,5 +1,6 @@
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use errors::AnyhowResult;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::dependencies::BvhToWorkflowDependencies;
 
 use crate::job::job_types::format_conversion::fbx_to_gltf::dependencies::FbxToGltfDependencies;
 use crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies;
@@ -28,6 +29,7 @@ pub struct JobSpecificDependencies {
   pub maybe_styletts2_dependencies: Option<StyleTTS2Dependencies>,
   pub maybe_comfy_ui_dependencies: Option<ComfyDependencies>,
   pub maybe_convert_fbx_to_gltf_dependencies: Option<FbxToGltfDependencies>,
+  pub maybe_convert_bvh_to_workflow_dependencies: Option<BvhToWorkflowDependencies>,
 }
 
 impl JobSpecificDependencies {
@@ -45,6 +47,7 @@ impl JobSpecificDependencies {
     let mut maybe_styletts2_dependencies = None;
     let mut maybe_comfy_ui_dependencies = None;
     let mut maybe_convert_fbx_to_gltf_dependencies = None;
+    let mut maybe_convert_bvh_to_workflow_dependencies = None;
 
     if scoped_execution.can_run_job(InferenceModelType::RvcV2) {
       print_with_space("Setting RVCv2 dependencies...");
@@ -106,6 +109,11 @@ impl JobSpecificDependencies {
       maybe_convert_fbx_to_gltf_dependencies = Some(FbxToGltfDependencies::setup()?);
     }
 
+    if scoped_execution.can_run_job(InferenceModelType::BvhToWorkflow) {
+      print_with_space("Setting ConvertBvhToWorkflow dependencies...");
+      maybe_convert_bvh_to_workflow_dependencies = Some(BvhToWorkflowDependencies::setup()?);
+    }
+
     Ok(JobSpecificDependencies {
       maybe_rvc_v2_dependencies,
       maybe_sad_talker_dependencies,
@@ -119,6 +127,7 @@ impl JobSpecificDependencies {
       maybe_styletts2_dependencies,
       maybe_comfy_ui_dependencies,
       maybe_convert_fbx_to_gltf_dependencies,
+      maybe_convert_bvh_to_workflow_dependencies,
     })
   }
 }

crates/service/job/inference_job/src/main.rs

@@ -22,6 +22,11 @@ use std::path::PathBuf;
 use std::time::Duration;
 
 use log::{info, warn};
+use opentelemetry::{KeyValue};
+use opentelemetry::metrics::Unit;
+use opentelemetry_otlp::WithExportConfig;
+use opentelemetry_sdk::metrics::reader::{DefaultAggregationSelector, DefaultTemporalitySelector};
+use opentelemetry_sdk::Resource;
 use r2d2_redis::r2d2;
 use r2d2_redis::RedisConnectionManager;
 use sqlx::mysql::MySqlPoolOptions;
@@ -51,6 +56,8 @@ use crate::http_server::run_http_server::launch_http_server;
 use crate::job::job_loop::main_loop::main_loop;
 use crate::job_dependencies::{BucketDependencies, ClientDependencies, DatabaseDependencies, FileSystemDetails, JobCaches, JobDependencies, JobInstanceInfo, JobSystemControls, JobSystemDependencies};
 use crate::job_specific_dependencies::JobSpecificDependencies;
+use crate::util::instrumentation::JobInstruments;
+use crate::util::model_weights_cache::model_weights_cache_directory::ModelWeightsCacheDirectory;
 use crate::util::scoped_execution::ScopedExecution;
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
 
@@ -72,10 +79,34 @@ const ENV_PUBLIC_BUCKET_NAME : &str = "PUBLIC_BUCKET_NAME";
 // HTTP sidecar
 const ENV_TTS_INFERENCE_SIDECAR_HOSTNAME: &str = "TTS_INFERENCE_SIDECAR_HOSTNAME";
 
+const OTEL_METER_NAME: &str = "inference-job";
+
+// TODO@madhukar93: labels for model etc
+fn init_otel_metrics_pipeline() -> Result<(), opentelemetry::metrics::MetricsError>  {
+ let provider = opentelemetry_otlp::new_pipeline()
+     .metrics(opentelemetry_sdk::runtime::Tokio)
+     // TODO: 1. read host from env 2. Single pod of otel-collector is probably not good enough, run daemonset?
+     .with_exporter(opentelemetry_otlp::new_exporter().tonic().with_endpoint("http://adot-collector.adot-collector-kubeprometheus:4317"))
+     .with_resource(Resource::new(vec![KeyValue::new("service.name", "inference-job")]))
+     .with_period(Duration::from_secs(3))
+     .with_timeout(Duration::from_secs(10))
+     .with_aggregation_selector(DefaultAggregationSelector::new())
+     .with_temporality_selector(DefaultTemporalitySelector::new())
+     .build()?;
+  opentelemetry::global::set_meter_provider(provider);
+  Ok(())
+}
+
 //#[tokio::main]
 #[actix_web::main]
 async fn main() -> AnyhowResult<()> {
 
+  if let Err(e) = init_otel_metrics_pipeline() {
+    warn!("Failed to initialize OpenTelemetry metrics pipeline, continuing execution: {}", e);
+  }
+
+  let meter = opentelemetry::global::meter("inference-job");
+
   let container_environment = bootstrap(BootstrapArgs {
     app_name: "inference-job",
     default_logging_override: Some(DEFAULT_RUST_LOG),
@@ -204,6 +235,10 @@ async fn main() -> AnyhowResult<()> {
 
   let job_specific_dependencies = JobSpecificDependencies::setup_for_jobs(&scoped_execution)?;
 
+  let scoped_tempdir_for_downloads = ScopedTempDirCreator::for_directory(
+    easyenv::get_env_pathbuf_or_default(
+      "SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived")));
+
   let job_dependencies = JobDependencies {
     db: DatabaseDependencies {
       mysql_pool,
@@ -215,13 +250,15 @@ async fn main() -> AnyhowResult<()> {
       scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory(
         easyenv::get_env_pathbuf_or_default(
           "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_short_lived"))),
-      scoped_temp_dir_creator_for_long_lived_downloads: ScopedTempDirCreator::for_directory(
-        easyenv::get_env_pathbuf_or_default(
-          "SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived"))),
+      scoped_temp_dir_creator_for_long_lived_downloads: scoped_tempdir_for_downloads.clone(),
       scoped_temp_dir_creator_for_work: ScopedTempDirCreator::for_directory(
         easyenv::get_env_pathbuf_or_default(
           "SCOPED_TEMP_DIR_WORK", PathBuf::from("/tmp/downloads_long_lived"))),
       semi_persistent_cache,
+      model_weights_cache_directory: ModelWeightsCacheDirectory::setup_from_env_and_deps(
+        &scoped_tempdir_for_downloads,
+        &public_bucket_client,
+      )?,
     },
     buckets: BucketDependencies {
       public_bucket_client,
@@ -277,6 +314,7 @@ async fn main() -> AnyhowResult<()> {
       },
       job_specific_dependencies,
     },
+    job_instruments: JobInstruments::new_from_meter(meter),
   };
 
   set_up_directories(&job_dependencies)?;

crates/service/job/inference_job/src/util/instrumentation.rs

@@ -0,0 +1,69 @@
+use opentelemetry::metrics::{Counter, Histogram, Meter, Unit, UpDownCounter};
+use opentelemetry_sdk::metrics::data::Gauge;
+
+pub struct JobInstruments {
+    pub job_duration: Histogram<u64>,
+    pub inference_command_execution_duration: Histogram<u64>,
+    pub batch_query_duration: Histogram<u64>,
+    pub batch_size: Histogram<u64>,
+    pub batch_processing_duration: Histogram<u64>,
+    pub total_job_count: Counter<u64>,
+    pub job_success_count: Counter<u64>,
+    pub job_failure_count: Counter<u64>,
+}
+
+impl JobInstruments {
+    pub fn new_from_meter(meter: Meter) -> Self {
+        let job_duration = meter.u64_histogram("inference_job_duration")
+            .with_unit(Unit::new("milliseconds"))
+            // job type could be a label, and this instrument could be common for all jobs
+            .with_description("inference job duration")
+            .init();
+
+        let batch_query_duration = meter.u64_histogram("inference_job_batch_query_duration")
+            .with_unit(Unit::new("milliseconds"))
+            .with_description("inference job batch query duration")
+            .init();
+
+        let batch_size = meter.u64_histogram("inference_job_batch_size")
+            .with_unit(Unit::new("jobs"))
+            .with_description("inference job batch size")
+            .init();
+
+        let inference_command_execution_duration = meter.u64_histogram("inference_job_inference_command_execution_duration")
+            .with_unit(Unit::new("milliseconds"))
+            .with_description("inference job inference command execution duration")
+            .init();
+
+        let batch_processing_duration = meter.u64_histogram("inference_job_batch_processing_duration")
+            .with_unit(Unit::new("milliseconds"))
+            .with_description("inference job batch processing duration")
+            .init();
+
+        let total_job_count = meter.u64_counter("inference_job_total_job_count")
+            .with_unit(Unit::new("jobs"))
+            .with_description("inference job total job count")
+            .init();
+
+        let job_success_count = meter.u64_counter("inference_job_success_count")
+            .with_unit(Unit::new("jobs"))
+            .with_description("inference job success count")
+            .init();
+
+        let job_failure_count = meter.u64_counter("inference_job_failure_count")
+            .with_unit(Unit::new("jobs"))
+            .with_description("inference job failure count")
+            .init();
+
+        Self {
+            job_duration,
+            inference_command_execution_duration,
+            batch_query_duration,
+            batch_size,
+            batch_processing_duration,
+            total_job_count,
+            job_success_count,
+            job_failure_count,
+        }
+    }
+}

crates/service/job/inference_job/src/util/mod.rs

@@ -1,7 +1,10 @@
 pub mod common_commands;
 pub mod file_cache_dir;
 pub mod from_env;
+pub mod instrumentation;
 pub mod maybe_download_file_from_bucket;
+pub mod model_weights_cache;
+
 #[macro_use]
 pub mod model_downloader;
 pub mod scoped_execution;

crates/service/job/inference_job/src/util/model_weights_cache/mod.rs

@@ -0,0 +1,2 @@
+pub mod model_weights_cache_directory;
+pub mod model_weights_cache_filename;

crates/service/job/inference_job/src/util/model_weights_cache/model_weights_cache_directory.rs

@@ -0,0 +1,200 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use log::{error, info, warn};
+
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+use cloud_storage::bucket_client::BucketClient;
+use errors::AnyhowResult;
+use filesys::file_size::file_size;
+use filesys::rename_across_devices::{rename_across_devices, RenameError};
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
+use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
+use mysql_queries::queries::model_weights::get::get_weight::RetrievedModelWeight;
+
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::util::model_weights_cache::model_weights_cache_filename::ModelWeightsCacheMapping;
+use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
+
+#[derive(Clone)]
+pub struct ModelWeightsCacheDirectory {
+  /// The path to the cache directory
+  pub local_cache_directory: PathBuf,
+
+  /// Create tempdirs during the download process.
+  pub scoped_tempdir_creator: ScopedTempDirCreator,
+
+  /// A client to download from the public bucket.
+  pub public_bucket_client: BucketClient,
+
+  /// NB: Sometimes the downloader incompletely downloads files. Typically, this is a zero file
+  /// size, but the Rust system may report a non-zero (but small) number of bytes. This should
+  /// later be investigated and the heuristic simplified. The intent here is merely to make sure
+  /// we don't consider these okay:
+  ///
+  ///   -rw-r--r-- 1 root root 55824433 Nov 30 00:12 VM:012rkwsv91zb
+  ///   -rw-r--r-- 1 root root 55823149 Nov 30 00:13 VM:79etbx4fdksv
+  ///   -rw-r--r-- 1 root root        0 Nov 30 00:52 VM:1dzepsnwzbkc (bad file)
+  ///   -rw-r--r-- 1 root root        0 Nov 30 00:29 VM:7c2df5a36qjb (bad file)
+  ///
+  pub maybe_cached_file_minimum_size_required: Option<u64>,
+
+  /// Update the status of jobs when we manage the cache.
+  pub maybe_job_progress_reporter: Option<Arc<dyn JobProgressReporter>>,
+}
+
+
+impl ModelWeightsCacheDirectory {
+  /// Configure and try to create the directory if it doesn't exist.
+  pub fn setup_from_env_and_deps(
+    scoped_temp_dir_creator: &ScopedTempDirCreator,
+    public_bucket_client: &BucketClient,
+  ) -> AnyhowResult<Self> {
+    let directory = easyenv::get_env_pathbuf_or_default(
+      "MODEL_WEIGHTS_CACHE_DIR",
+      "/model_weights_cache");
+
+    info!("Using model weights cache directory: {:?}", directory);
+
+    let directory = ModelWeightsCacheDirectory {
+      local_cache_directory: directory,
+      maybe_cached_file_minimum_size_required: Some(1000), // TODO(bt,2024-02-11): Make configurable
+      maybe_job_progress_reporter: None, // TODO(bt,2024-02-11): Make Sync/Send to hold a copy
+      scoped_tempdir_creator: scoped_temp_dir_creator.clone(),
+      public_bucket_client: public_bucket_client.clone(),
+    };
+
+    directory.try_create_directory()?;
+
+    Ok(directory)
+  }
+
+
+  /// Try to create the directory if it doesn't exist.
+  fn try_create_directory(&self) -> std::io::Result<()> {
+    if !self.local_cache_directory.exists() {
+      std::fs::create_dir_all(&self.local_cache_directory)?;
+    }
+    return Ok(())
+  }
+
+
+  pub fn model_weight_cache_path(&self, model_weight: &RetrievedModelWeight) -> PathBuf {
+    let filename = ModelWeightsCacheMapping::new_from_model(model_weight);
+    let full_path = self.local_cache_directory.join(filename.to_path_buf());
+    full_path
+  }
+
+
+  /// Download the model if we don't already have a copy.
+  /// Return the path from the cache.
+  pub async fn get_model_weight_from_cache_or_bucket(
+    &self,
+    model_weight: &RetrievedModelWeight
+  ) -> Result<PathBuf, ProcessSingleJobError> {
+    let cache_filesystem_path = self.model_weight_cache_path(model_weight);
+
+    if self.is_model_already_cached(model_weight, &cache_filesystem_path)? {
+      return Ok(cache_filesystem_path);
+    }
+
+    self.report_progress(format!("downloading model {}", &model_weight.token)).await;
+
+    let bucket_object_path = WeightFileBucketPath::from_object_hash(
+      &model_weight.public_bucket_hash,
+      model_weight.maybe_public_bucket_prefix.as_deref(),
+      model_weight.maybe_public_bucket_extension.as_deref(),
+    );
+
+    let bucket_object_path = bucket_object_path.to_full_object_pathbuf();
+
+    info!("Downloading model weight (type {:?} token {:?}) from bucket object path {:?}",
+        &model_weight.weights_type,
+        &model_weight.token,
+        &bucket_object_path);
+
+    // NB (1): Download to temp directory to stop concurrent writes and race conditions from other
+    //         workers writing to a shared volume.
+    // NB (2): TempDir exists until it goes out of scope, at which point it should delete from filesystem.
+    let temp_dir = self.scoped_tempdir_creator.new_tempdir("model_weight_download")
+        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+    let temp_path = temp_dir.path().join("download.part");
+
+    self.public_bucket_client.download_file_to_disk(&bucket_object_path, &temp_path)
+        .await
+        .map_err(|e| {
+          safe_delete_temp_directory(&temp_dir);
+          ProcessSingleJobError::Other(e)
+        })?;
+
+    info!("Downloaded successfully from bucket!");
+
+    let original_size = file_size(&temp_path)
+        .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+
+    info!("File size of temp download file {:?} is {original_size}", &temp_path);
+
+    info!("Renaming temp file from {:?} to {:?}!", &temp_path, &cache_filesystem_path);
+
+    rename_across_devices(&temp_path, &cache_filesystem_path)
+        .map_err(|err| {
+          error!("could not rename on disk: {:?}", err);
+          safe_delete_temp_file(&temp_path);
+          safe_delete_temp_directory(&temp_dir);
+          match err {
+            RenameError::StorageFull => ProcessSingleJobError::FilesystemFull,
+            RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err),
+          }
+        })?;
+
+    info!("Finished downloading and moving file to {:?}", &cache_filesystem_path);
+
+    safe_delete_temp_file(&temp_path);
+    safe_delete_temp_directory(&temp_dir);
+
+    Ok(cache_filesystem_path)
+  }
+
+
+  fn is_model_already_cached(&self, model_weight: &RetrievedModelWeight, full_cache_filesystem_path: &PathBuf) -> Result<bool, ProcessSingleJobError> {
+    if !full_cache_filesystem_path.exists() {
+      warn!("Model weight (type {:?} token {:?}) does not already exist at path {:?}",
+        &model_weight.weights_type,
+        &model_weight.token,
+        &full_cache_filesystem_path);
+
+      return Ok(false);
+    }
+
+    info!("Model weight (type {:?} token {:?}) exists at path {:?}",
+      &model_weight.weights_type,
+      &model_weight.token,
+      &full_cache_filesystem_path);
+
+    if let Some(existing_file_minimum_size_required) = self.maybe_cached_file_minimum_size_required {
+      let size = file_size(full_cache_filesystem_path)
+          .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+
+      info!("Model weight has file size = {size}");
+
+      if size < existing_file_minimum_size_required {
+        return Ok(false); // NB: We want to re-download the file.
+      }
+    }
+
+    // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity
+    Ok(true)
+  }
+
+
+  async fn report_progress(&self, _description: String) {
+    if let Some(_job_progress_reporter) = &self.maybe_job_progress_reporter {
+      // TODO(bt,2024-02-10): Needs interior mutability. Meh. Also fail open.
+      //job_progress_reporter.log_status(&description)
+      //    .map_err(|e| ProcessSingleJobError::Other(e))?;
+    }
+  }
+}

crates/service/job/inference_job/src/util/model_weights_cache/model_weights_cache_filename.rs

@@ -0,0 +1,78 @@
+use std::path::PathBuf;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use mysql_queries::queries::model_weights::get::get_weight::RetrievedModelWeight;
+use tokens::tokens::model_weights::ModelWeightToken;
+
+pub struct ModelWeightsCacheMapping {
+  pub token: ModelWeightToken,
+  pub weights_type: WeightsType,
+}
+
+impl ModelWeightsCacheMapping {
+
+  pub fn new_from_model(model_weight: &RetrievedModelWeight) -> Self {
+    Self::from_token_and_type(&model_weight.token, model_weight.weights_type)
+  }
+
+  pub fn from_token_and_type(token: &ModelWeightToken, weights_type: WeightsType) -> Self {
+    Self {
+      token: token.clone(),
+      weights_type,
+    }
+  }
+
+
+  pub fn to_path_buf(&self) -> PathBuf {
+    let extension = self.extension();
+    let filename = format!("{}.{}.{}", &self.token, self.weights_type, extension);
+    PathBuf::from(filename)
+  }
+
+
+  fn extension(&self) -> &str {
+    match self.weights_type {
+      // Pt
+      WeightsType::HifiganTacotron2 => "pt",
+      WeightsType::RvcV2 => "pt",
+      WeightsType::SoVitsSvc => "pt",
+      WeightsType::Tacotron2 => "pt",
+      WeightsType::VallE => "pt",
+      // Safetensors
+      WeightsType::StableDiffusion15 => "safetensors",
+      WeightsType::StableDiffusionXL => "safetensors",
+      WeightsType::LoRA => "safetensors",
+      // Json
+      WeightsType::ComfyUi => "json",
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use std::path::PathBuf;
+  use enums::by_table::model_weights::weights_types::WeightsType;
+  use tokens::tokens::model_weights::ModelWeightToken;
+  use crate::util::model_weights_cache::model_weights_cache_filename::ModelWeightsCacheMapping;
+
+  #[test]
+  fn stable_diffusion_checkpoints() {
+    let mapper = ModelWeightsCacheMapping::from_token_and_type(
+      &ModelWeightToken::new_from_str("test_token"),
+      WeightsType::StableDiffusion15
+    );
+    let path = mapper.to_path_buf();
+
+    assert_eq!(path, PathBuf::from("test_token.sd_1.5.safetensors"));
+  }
+
+  #[test]
+  fn lora_checkpoints() {
+    let mapper = ModelWeightsCacheMapping::from_token_and_type(
+      &ModelWeightToken::new_from_str("test_lora"),
+      WeightsType::LoRA
+    );
+    let path = mapper.to_path_buf();
+
+    assert_eq!(path, PathBuf::from("test_lora.loRA.safetensors"));
+  }
+}

crates/service/plugins/users/Cargo.toml

@@ -69,3 +69,6 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls",
 r2d2_redis = "0.14.0"
 byteorder = "1.5.0"
 strum_macros = "0.25.3"
+
+utoipa = { version = "4", features = ["chrono","actix_extras"] }
+utoipa-swagger-ui = { version = "5", features = ["actix-web"] }

crates/service/plugins/users/src/common_responses/mod.rs

@@ -0,0 +1,2 @@
+pub mod user_avatars;
+pub mod user_details_lite;

crates/service/plugins/users/src/common_responses/user_avatars/default_avatar_color_from_username.rs

@@ -33,7 +33,7 @@ mod tests {
 
   use rand::distributions::{Alphanumeric, DistString};
 
-  use crate::util::placeholder_images::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username;
+  use crate::common_responses::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username;
 
   #[test]
   fn test_stability() {

crates/service/plugins/users/src/common_responses/user_avatars/default_avatar_from_username.rs

@@ -32,7 +32,7 @@ mod tests {
 
   use rand::distributions::{Alphanumeric, DistString};
 
-  use crate::util::placeholder_images::user_avatars::default_avatar_from_username::default_avatar_from_username;
+  use crate::common_responses::user_avatars::default_avatar_from_username::default_avatar_from_username;
 
   #[test]
   fn test_stability() {

crates/service/plugins/users/src/common_responses/user_details_lite.rs

@@ -2,8 +2,8 @@ use utoipa::ToSchema;
 
 use tokens::tokens::users::UserToken;
 
-use crate::util::placeholder_images::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username;
-use crate::util::placeholder_images::user_avatars::default_avatar_from_username::default_avatar_from_username;
+use crate::common_responses::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username;
+use crate::common_responses::user_avatars::default_avatar_from_username::default_avatar_from_username;
 
 /// Everything we need to refer to a user on the public web interface.
 #[derive(Clone, Serialize, ToSchema)]
@@ -107,7 +107,7 @@ impl DefaultAvatarInfo {
 mod tests {
   use tokens::tokens::users::UserToken;
 
-  use crate::http_server::common_responses::user_details_lite::UserDetailsLight;
+  use crate::common_responses::user_details_lite::UserDetailsLight;
 
   #[test]
   fn test_from_optional_db_fields() {

crates/service/plugins/users/src/endpoints/get_profile_handler.rs

@@ -15,6 +15,7 @@ use r2d2_redis::{r2d2, RedisConnectionManager};
 use r2d2_redis::r2d2::PooledConnection;
 use r2d2_redis::redis::Commands;
 use sqlx::MySqlPool;
+use utoipa::ToSchema;
 
 use enums::common::visibility::Visibility;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
@@ -25,15 +26,19 @@ use mysql_queries::queries::users::user_badges::list_user_badges::UserBadgeForLi
 use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::{get_user_profile_by_username_from_connection, UserProfileResult};
 use tokens::tokens::users::UserToken;
 
-use crate::utils::default_avatar_color_from_username::default_avatar_color_from_username;
-use crate::utils::default_avatar_from_username::default_avatar_from_username;
+use crate::common_responses::user_avatars::default_avatar_color_from_username::default_avatar_color_from_username;
+use crate::common_responses::user_avatars::default_avatar_from_username::default_avatar_from_username;
+use crate::common_responses::user_details_lite::UserDetailsLight;
 use crate::utils::session_checker::SessionChecker;
 
 // TODO: This is duplicated in query_user_profile
 
-#[derive(Serialize)]
+#[derive(Serialize, ToSchema)]
 pub struct UserProfileRecordForResponse {
   pub user_token: UserToken,
+
+  pub core_info: UserDetailsLight,
+  
   pub username: String,
   pub display_name: String,
   pub email_gravatar_hash: String,
@@ -52,32 +57,41 @@ pub struct UserProfileRecordForResponse {
   pub github_username: Option<String>,
   pub cashapp_username: Option<String>,
   pub website_url: Option<String>,
-  pub badges: Vec<UserBadgeForList>,
+  pub badges: Vec<UserProfileUserBadge>,
   pub created_at: DateTime<Utc>,
   pub maybe_moderator_fields: Option<UserProfileModeratorFields>,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, ToSchema)]
 pub struct UserProfileModeratorFields {
   pub is_banned: bool,
   pub maybe_mod_comments: Option<String>,
   pub maybe_mod_user_token: Option<String>,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Deserialize, Clone, ToSchema)]
+pub struct UserProfileUserBadge {
+  pub slug: String,
+  pub title: String,
+  pub description: String,
+  pub image_url: String,
+  pub granted_at: DateTime<Utc>,
+}
+
+#[derive(Serialize, ToSchema)]
 pub struct ProfileSuccessResponse {
   pub success: bool,
   pub user: Option<UserProfileRecordForResponse>,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, ToSchema)]
 pub enum ProfileError {
   ServerError,
   NotFound,
 }
 
 /// For the URL PathInfo
-#[derive(Deserialize)]
+#[derive(Deserialize, ToSchema)]
 pub struct GetProfilePathInfo {
   username: String,
 }
@@ -102,6 +116,18 @@ impl fmt::Display for ProfileError {
   }
 }
 
+#[utoipa::path(
+  get,
+  path = "/user/{username}/profile",
+  responses(
+    (status = 200, description = "Get profile", body = UserProfileRecordForResponse),
+    (status = 404, description = "Not found", body = ProfileError),
+    (status = 500, description = "Server error", body = ProfileError),
+  ),
+  params(
+    ("path" = GetProfilePathInfo, description = "Path for Request")
+  )
+)]
 pub async fn get_profile_handler(
   http_request: HttpRequest,
   path: Path<GetProfilePathInfo>,
@@ -219,7 +245,13 @@ pub async fn get_profile_handler(
   }
 
   let mut profile_for_response = UserProfileRecordForResponse {
-    user_token: user_data.user_profile.user_token,
+    user_token: user_data.user_profile.user_token.clone(), // NB: Cloned because of ref use for avatar below
+    core_info: UserDetailsLight::from_db_fields(
+      &user_data.user_profile.user_token,
+      &user_data.user_profile.username,
+      &user_data.user_profile.display_name,
+      &user_data.user_profile.email_gravatar_hash,
+    ),
     username: user_data.user_profile.username.to_string(), // NB: Cloned because of ref use for avatar below
     display_name: user_data.user_profile.display_name,
     email_gravatar_hash: user_data.user_profile.email_gravatar_hash,
@@ -246,7 +278,15 @@ pub async fn get_profile_handler(
         maybe_mod_user_token: mod_fields.maybe_mod_user_token,
       }
     }),
-    badges: user_data.badges,
+    badges: user_data.badges
+        .into_iter()
+        .map(|badge| UserProfileUserBadge {
+          slug: badge.slug,
+          title: badge.title,
+          description: badge.description,
+          image_url: badge.image_url,
+          granted_at: badge.granted_at,
+        }).collect(),
   };
 
   if !is_mod {

crates/service/plugins/users/src/endpoints/session_info_handler.rs

@@ -10,13 +10,15 @@ use actix_web::error::ResponseError;
 use actix_web::http::StatusCode;
 use log::warn;
 use sqlx::MySqlPool;
+use utoipa::ToSchema;
 
 use http_server_common::response::response_error_helpers::to_simple_json_error;
 
+use crate::common_responses::user_details_lite::UserDetailsLight;
 use crate::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager;
 use crate::utils::session_checker::SessionChecker;
 
-#[derive(Serialize, Copy, Clone)]
+#[derive(Serialize, Copy, Clone, ToSchema)]
 #[serde(rename_all = "snake_case")]
 pub enum FakeYouPlan {
   Free,
@@ -25,7 +27,7 @@ pub enum FakeYouPlan {
   Pro,
 }
 
-#[derive(Serialize, Copy, Clone)]
+#[derive(Serialize, Copy, Clone, ToSchema)]
 #[serde(rename_all = "snake_case")]
 pub enum StorytellerStreamPlan {
   Free,
@@ -34,13 +36,18 @@ pub enum StorytellerStreamPlan {
   Pro,
 }
 
-#[derive(Serialize)]
-pub struct UserInfo {
+#[derive(Serialize, ToSchema)]
+pub struct SessionUserInfo {
+  pub core_info: UserDetailsLight,
+
   pub user_token: String,
   pub username: String,
   pub display_name: String,
   pub email_gravatar_hash: String,
 
+  // Feature / rollout flags:
+  pub can_access_studio: bool,
+
   // Premium plans:
   pub fakeyou_plan: FakeYouPlan,
   pub storyteller_stream_plan: StorytellerStreamPlan,
@@ -71,14 +78,14 @@ pub struct UserInfo {
   pub can_delete_users: bool,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, ToSchema)]
 pub struct SessionInfoSuccessResponse {
   pub success: bool,
   pub logged_in: bool,
-  pub user: Option<UserInfo>,
+  pub user: Option<SessionUserInfo>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, ToSchema)]
 pub enum SessionInfoError {
   ServerError,
 }
@@ -106,6 +113,14 @@ impl fmt::Display for SessionInfoError {
   }
 }
 
+#[utoipa::path(
+  get,
+  path = "/v1/session",
+  responses(
+    (status = 200, description = "Get profile", body = SessionInfoSuccessResponse),
+    (status = 500, description = "Server error", body = SessionInfoError),
+  ),
+)]
 pub async fn session_info_handler(
   http_request: HttpRequest,
   mysql_pool: web::Data<MySqlPool>,
@@ -137,12 +152,21 @@ pub async fn session_info_handler(
       if !session_data.is_banned {
         // NB: Banned users can't be logged in
         logged_in = true;
-        user_info = Some(UserInfo {
+        user_info = Some(SessionUserInfo {
+          core_info: UserDetailsLight::from_db_fields(
+            &session_data.user_token_typed,
+            &session_data.username,
+            &session_data.display_name,
+            &session_data.email_gravatar_hash,
+          ),
           user_token: session_data.user_token.clone(),
           username: session_data.username.to_string(),
           display_name: session_data.display_name.to_string(),
           email_gravatar_hash: session_data.email_gravatar_hash.to_string(),
 
+          // Rollout / feature flags:
+          can_access_studio: session_data.can_access_studio,
+
           // Premium plans:
           fakeyou_plan: FakeYouPlan::Free,
           storyteller_stream_plan: StorytellerStreamPlan::Free,

crates/service/plugins/users/src/lib.rs

@@ -4,6 +4,7 @@
 pub const RESERVED_USERNAMES : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames.txt");
 pub const RESERVED_SUBSTRINGS : &str = include_str!("../../../../../includes/binary_includes/reserved_usernames_including.txt");
 
+pub mod common_responses;
 pub mod cookies;
 pub mod default_routes;
 pub mod endpoints;

crates/service/plugins/users/src/utils/default_avatar_color_from_username.rs

@@ -1,70 +0,0 @@
-use std::collections::hash_map::DefaultHasher;
-use std::hash::{Hash, Hasher};
-
-// TODO: De-duplicate with storyteller-web
-
-/// The frontend folks will tell us how many colors to use.
-const NUMBER_OF_COLORS : u64 = 8;
-
-/// Not that it matters, but this perturbs the hash.
-const SALT_LIKE_OFFSET : u8 = 51;
-
-/// We return an index instead of a color, that way the frontend can drive.
-/// The hash should be stable with respect to username.
-pub fn default_avatar_color_from_username(username: &str) -> u8 {
-  // We don't want to accidentally let case influence the output.
-  // There's a potential of mixing up `username` and `display_name`.
-  let username_lower = username.to_lowercase();
-
-  let mut hasher = DefaultHasher::new();
-
-  username_lower.hash(&mut hasher);
-  SALT_LIKE_OFFSET.hash(&mut hasher);
-
-  let hash = hasher.finish();
-
-  let color_index = hash % NUMBER_OF_COLORS;
-  color_index as u8
-}
-
-#[cfg(test)]
-mod tests {
-  use std::collections::HashSet;
-
-  use rand::distributions::{Alphanumeric, DistString};
-
-  use crate::utils::default_avatar_color_from_username::default_avatar_color_from_username;
-
-  #[test]
-  fn test_stability() {
-    assert_eq!(default_avatar_color_from_username("echelon"), 3);
-    assert_eq!(default_avatar_color_from_username("vegito"), 4);
-    assert_eq!(default_avatar_color_from_username("bflat"), 1);
-    assert_eq!(default_avatar_color_from_username("mechacosm"), 1);
-    assert_eq!(default_avatar_color_from_username("sajattack"), 3);
-    assert_eq!(default_avatar_color_from_username("zdisket"), 7);
-  }
-
-  #[test]
-  fn test_case_insensitivity() {
-    assert_eq!(default_avatar_color_from_username("echelon"), 3);
-    assert_eq!(default_avatar_color_from_username("ECHELON"), 3);
-    assert_eq!(default_avatar_color_from_username("EcHeLoN"), 3);
-    assert_eq!(default_avatar_color_from_username("ECHElon"), 3);
-    assert_eq!(default_avatar_color_from_username("echELON"), 3);
-  }
-
-  #[test]
-  fn test_range() {
-    let mut distribution = HashSet::new();
-    for _ in 0..1000 {
-      let random_username = Alphanumeric.sample_string(&mut rand::thread_rng(), 16);
-      let avatar_id = default_avatar_color_from_username(&random_username);
-      distribution.insert(avatar_id);
-      assert!(avatar_id >= 0);
-      assert!(avatar_id <= 8);
-    }
-    // NB: We could test frequency of the distribution too
-    assert_eq!(distribution.len(), 8);
-  }
-}

crates/service/plugins/users/src/utils/default_avatar_from_username.rs

@@ -1,71 +0,0 @@
-use std::collections::hash_map::DefaultHasher;
-use std::hash::{Hash, Hasher};
-
-// TODO: De-duplicate with storyteller-web
-
-/// There are currently 25 avatars numbered 0 to 24 (0-indexed).
-/// The original dataset was numbered 1 - 25, but I renamed 25 to 0.
-const NUMBER_OF_AVATARS : u64 = 25;
-
-/// Not that it matters, but this perturbs the hash.
-const SALT_LIKE_OFFSET : u8 = 21;
-
-/// We return an index instead of a filename, that way the frontend can drive.
-/// The hash should be stable with respect to username.
-pub fn default_avatar_from_username(username: &str) -> u8 {
-  // We don't want to accidentally let case influence the output.
-  // There's a potential of mixing up `username` and `display_name`.
-  let username_lower = username.to_lowercase();
-
-  let mut hasher = DefaultHasher::new();
-
-  username_lower.hash(&mut hasher);
-  SALT_LIKE_OFFSET.hash(&mut hasher);
-
-  let hash = hasher.finish();
-
-  let avatar_index= hash % NUMBER_OF_AVATARS;
-  avatar_index as u8
-}
-
-#[cfg(test)]
-mod tests {
-  use std::collections::HashSet;
-
-  use rand::distributions::{Alphanumeric, DistString};
-
-  use crate::utils::default_avatar_from_username::default_avatar_from_username;
-
-  #[test]
-  fn test_stability() {
-    assert_eq!(default_avatar_from_username("echelon"), 10);
-    assert_eq!(default_avatar_from_username("vegito"), 20);
-    assert_eq!(default_avatar_from_username("bflat"), 22);
-    assert_eq!(default_avatar_from_username("mechacosm"), 15);
-    assert_eq!(default_avatar_from_username("sajattack"), 15);
-    assert_eq!(default_avatar_from_username("zdisket"), 19);
-  }
-
-  #[test]
-  fn test_case_insensitivity() {
-    assert_eq!(default_avatar_from_username("echelon"), 10);
-    assert_eq!(default_avatar_from_username("ECHELON"), 10);
-    assert_eq!(default_avatar_from_username("EcHeLoN"), 10);
-    assert_eq!(default_avatar_from_username("ECHElon"), 10);
-    assert_eq!(default_avatar_from_username("echELON"), 10);
-  }
-
-  #[test]
-  fn test_range() {
-    let mut distribution = HashSet::new();
-    for _ in 0..1000 {
-      let random_username = Alphanumeric.sample_string(&mut rand::thread_rng(), 16);
-      let avatar_id = default_avatar_from_username(&random_username);
-      distribution.insert(avatar_id);
-      assert!(avatar_id >= 0);
-      assert!(avatar_id <= 24);
-    }
-    // NB: We could test frequency of the distribution too
-    assert_eq!(distribution.len(), 25);
-  }
-}