remove excess message

new dependencies + download workflow from weight token
save job args to file modification info
2026-10-30 15:25:57 -04:00 · 2024-02-20 07:42:27 -08:00 · 2024-02-20 07:41:05 -08:00 · 2024-02-17 06:21:51 -08:00 · 2024-02-14 19:12:18 -05:00 · 2024-02-14 18:52:45 -05:00
177 changed files with 4569 additions and 1490 deletions
@@ -78,4 +78,5 @@ TTS_HIFIGAN_SUPERRES_VOCODER_MODEL_FILENAME=hifigan_superres_twilight_33000.pt
 # Development key. Permissible to check in.
 NEWRELIC_API_KEY=c402be1c4c4216ec83d0cfc0851520cfFFFFNRAL

-STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting.
+# DO NOT UNCOMMENT THIS :P :P :P
+#STORYTELLER_ROOT="/home/tensor/code/storyteller/" # Put this in .env-secrets so that not everyone gets this setting.
@@ -17,6 +17,7 @@ production_database_voice_clone_report_download.sh
 secrets.toml
 secrets.yaml
 twitch_secrets.toml
+.DS_Store

 # Old env secrets
 .*-secrets
@@ -157,7 +157,7 @@
        "ordinal": 15,
        "name": "file_size_bytes",
        "type_info": {
-          "type": "Long",
+          "type": "LongLong",
          "flags": "NOT_NULL",
          "char_set": 63,
          "max_size": 10
@@ -288,7 +288,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -298,7 +298,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -308,7 +308,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -1,6 +1,6 @@
 {
  "db_name": "MySQL",
-  "query": "\nSELECT\n    users.token as user_token,\n    users.username,\n    users.display_name,\n\n    users.email_address,\n    users.email_confirmed,\n    users.email_gravatar_hash,\n\n    users.maybe_stripe_customer_id,\n    users.maybe_loyalty_program_key,\n\n    users.disable_gravatar,\n    users.auto_play_audio_preference,\n    users.auto_play_video_preference,\n    users.preferred_tts_result_visibility as `preferred_tts_result_visibility: enums::common::visibility::Visibility`,\n    users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: enums::common::visibility::Visibility`,\n\n    users.user_role_slug,\n    users.is_banned,\n\n    user_roles.can_use_tts,\n    user_roles.can_use_w2l,\n    user_roles.can_delete_own_tts_results,\n    user_roles.can_delete_own_w2l_results,\n    user_roles.can_delete_own_account,\n\n    user_roles.can_upload_tts_models,\n    user_roles.can_upload_w2l_templates,\n    user_roles.can_delete_own_tts_models,\n    user_roles.can_delete_own_w2l_templates,\n\n    user_roles.can_approve_w2l_templates,\n    user_roles.can_edit_other_users_profiles,\n    user_roles.can_edit_other_users_tts_models,\n    user_roles.can_edit_other_users_w2l_templates,\n    user_roles.can_delete_other_users_tts_models,\n    user_roles.can_delete_other_users_tts_results,\n    user_roles.can_delete_other_users_w2l_templates,\n    user_roles.can_delete_other_users_w2l_results,\n    user_roles.can_ban_users,\n    user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n    ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n    ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n    AND user_sessions.deleted_at IS NULL\n    AND users.user_deleted_at IS NULL\n    AND users.mod_deleted_at IS NULL\n        ",
+  "query": "\nSELECT\n    users.token as user_token,\n    users.username,\n    users.display_name,\n\n    users.email_address,\n    users.email_confirmed,\n    users.email_gravatar_hash,\n\n    users.maybe_stripe_customer_id,\n    users.maybe_loyalty_program_key,\n\n    users.disable_gravatar,\n    users.auto_play_audio_preference,\n    users.auto_play_video_preference,\n    users.preferred_tts_result_visibility as `preferred_tts_result_visibility: enums::common::visibility::Visibility`,\n    users.preferred_w2l_result_visibility as `preferred_w2l_result_visibility: enums::common::visibility::Visibility`,\n\n    users.user_role_slug,\n    users.is_banned,\n\n    users.can_access_studio,\n\n    user_roles.can_use_tts,\n    user_roles.can_use_w2l,\n    user_roles.can_delete_own_tts_results,\n    user_roles.can_delete_own_w2l_results,\n    user_roles.can_delete_own_account,\n\n    user_roles.can_upload_tts_models,\n    user_roles.can_upload_w2l_templates,\n    user_roles.can_delete_own_tts_models,\n    user_roles.can_delete_own_w2l_templates,\n\n    user_roles.can_approve_w2l_templates,\n    user_roles.can_edit_other_users_profiles,\n    user_roles.can_edit_other_users_tts_models,\n    user_roles.can_edit_other_users_w2l_templates,\n    user_roles.can_delete_other_users_tts_models,\n    user_roles.can_delete_other_users_tts_results,\n    user_roles.can_delete_other_users_w2l_templates,\n    user_roles.can_delete_other_users_w2l_results,\n    user_roles.can_ban_users,\n    user_roles.can_delete_users\n\nFROM users\nLEFT OUTER JOIN user_sessions\n    ON users.token = user_sessions.user_token\nLEFT OUTER JOIN user_roles\n    ON users.user_role_slug = user_roles.slug\nWHERE user_sessions.token = ?\n    AND user_sessions.deleted_at IS NULL\n    AND users.user_deleted_at IS NULL\n    AND users.mod_deleted_at IS NULL\n        ",
  "describe": {
    "columns": [
      {
@@ -155,6 +155,16 @@
      },
      {
        "ordinal": 15,
+        "name": "can_access_studio",
+        "type_info": {
+          "type": "Tiny",
+          "flags": "NOT_NULL | MULTIPLE_KEY",
+          "char_set": 63,
+          "max_size": 1
+        }
+      },
+      {
+        "ordinal": 16,
        "name": "can_use_tts",
        "type_info": {
          "type": "Tiny",
@@ -164,7 +174,7 @@
        }
      },
      {
-        "ordinal": 16,
+        "ordinal": 17,
        "name": "can_use_w2l",
        "type_info": {
          "type": "Tiny",
@@ -174,7 +184,7 @@
        }
      },
      {
-        "ordinal": 17,
+        "ordinal": 18,
        "name": "can_delete_own_tts_results",
        "type_info": {
          "type": "Tiny",
@@ -184,7 +194,7 @@
        }
      },
      {
-        "ordinal": 18,
+        "ordinal": 19,
        "name": "can_delete_own_w2l_results",
        "type_info": {
          "type": "Tiny",
@@ -194,7 +204,7 @@
        }
      },
      {
-        "ordinal": 19,
+        "ordinal": 20,
        "name": "can_delete_own_account",
        "type_info": {
          "type": "Tiny",
@@ -204,7 +214,7 @@
        }
      },
      {
-        "ordinal": 20,
+        "ordinal": 21,
        "name": "can_upload_tts_models",
        "type_info": {
          "type": "Tiny",
@@ -214,7 +224,7 @@
        }
      },
      {
-        "ordinal": 21,
+        "ordinal": 22,
        "name": "can_upload_w2l_templates",
        "type_info": {
          "type": "Tiny",
@@ -224,7 +234,7 @@
        }
      },
      {
-        "ordinal": 22,
+        "ordinal": 23,
        "name": "can_delete_own_tts_models",
        "type_info": {
          "type": "Tiny",
@@ -234,7 +244,7 @@
        }
      },
      {
-        "ordinal": 23,
+        "ordinal": 24,
        "name": "can_delete_own_w2l_templates",
        "type_info": {
          "type": "Tiny",
@@ -244,7 +254,7 @@
        }
      },
      {
-        "ordinal": 24,
+        "ordinal": 25,
        "name": "can_approve_w2l_templates",
        "type_info": {
          "type": "Tiny",
@@ -254,7 +264,7 @@
        }
      },
      {
-        "ordinal": 25,
+        "ordinal": 26,
        "name": "can_edit_other_users_profiles",
        "type_info": {
          "type": "Tiny",
@@ -264,7 +274,7 @@
        }
      },
      {
-        "ordinal": 26,
+        "ordinal": 27,
        "name": "can_edit_other_users_tts_models",
        "type_info": {
          "type": "Tiny",
@@ -274,7 +284,7 @@
        }
      },
      {
-        "ordinal": 27,
+        "ordinal": 28,
        "name": "can_edit_other_users_w2l_templates",
        "type_info": {
          "type": "Tiny",
@@ -284,7 +294,7 @@
        }
      },
      {
-        "ordinal": 28,
+        "ordinal": 29,
        "name": "can_delete_other_users_tts_models",
        "type_info": {
          "type": "Tiny",
@@ -294,7 +304,7 @@
        }
      },
      {
-        "ordinal": 29,
+        "ordinal": 30,
        "name": "can_delete_other_users_tts_results",
        "type_info": {
          "type": "Tiny",
@@ -304,7 +314,7 @@
        }
      },
      {
-        "ordinal": 30,
+        "ordinal": 31,
        "name": "can_delete_other_users_w2l_templates",
        "type_info": {
          "type": "Tiny",
@@ -314,7 +324,7 @@
        }
      },
      {
-        "ordinal": 31,
+        "ordinal": 32,
        "name": "can_delete_other_users_w2l_results",
        "type_info": {
          "type": "Tiny",
@@ -324,7 +334,7 @@
        }
      },
      {
-        "ordinal": 32,
+        "ordinal": 33,
        "name": "can_ban_users",
        "type_info": {
          "type": "Tiny",
@@ -334,7 +344,7 @@
        }
      },
      {
-        "ordinal": 33,
+        "ordinal": 34,
        "name": "can_delete_users",
        "type_info": {
          "type": "Tiny",
@@ -363,6 +373,7 @@
      false,
      false,
      false,
+      false,
      true,
      true,
      true,
@@ -384,5 +395,5 @@
      true
    ]
  },
-  "hash": "07d330b4c2e6b0dc76e2e61a3b44b47b8d287adf5eb0646f6cf2876655581a13"
+  "hash": "423109f2dcee23a558c43ecaab0eb2836714d5316620daaa1127b447cad59212"
 }
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  inference_category = ?,\n  maybe_model_type = ?,\n  maybe_model_token = ?,\n\n  maybe_input_source_token = ?,\n  maybe_input_source_token_type = ?,\n\n  maybe_download_url = ?,\n\n  maybe_raw_inference_text = ?,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  priority_level = ?,\n  is_keepalive_required = ?,\n  max_duration_seconds = ?,\n\n  is_debug_request = ?,\n  maybe_routing_tag = ?,\n\n  status = \"pending\"\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 20
+    },
+    "nullable": []
+  },
+  "hash": "4673b503ee7c44661b1d496aafc07cda7be410af1d46f3eeceee6895f3a07037"
+}
@@ -118,7 +118,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\n    INSERT INTO media_files\n    SET\n      token = ?,\n    \n      origin_category = ?,\n      origin_product_category = ?,\n      maybe_origin_model_type = ?,\n    \n      media_type = ?,\n      maybe_mime_type = ?,\n      file_size_bytes = ?,\n    \n      checksum_sha2 = ?,\n    \n      public_bucket_directory_hash = ?,\n      maybe_public_bucket_prefix = ?,\n      maybe_public_bucket_extension = ?,\n    \n      maybe_creator_user_token = ?,\n      maybe_creator_anonymous_visitor_token = ?,\n      creator_ip_address = ?,\n    \n      creator_set_visibility = ?,\n    \n      maybe_creator_file_synthetic_id = ?,\n      maybe_creator_category_synthetic_id = ?,\n    \n      is_generated_on_prem = ?,\n      generated_by_worker = ?,\n      generated_by_cluster = ?\n    \n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 20
-    },
-    "nullable": []
-  },
-  "hash": "567937a5918833330b74f94724eb69628aae49705ec4f8028321c38625eaa60c"
-}
@@ -118,7 +118,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -0,0 +1,47 @@
+{
+  "db_name": "MySQL",
+  "query": "\n      SELECT\n        token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n        weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n        updated_at\n      FROM model_weights\n      WHERE\n      updated_at > ?\n    ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::model_weights::ModelWeightToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "weights_type: enums::by_table::model_weights::weights_types::WeightsType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "5c67010afd0bb8aa3cd8f6c2407c4c314ace016f816eca51991720a276340d41"
+}
@@ -1,6 +1,6 @@
 {
  "db_name": "MySQL",
-  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    jobs.attempt_count,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n\n    jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n    jobs.maybe_model_type,\n    jobs.maybe_model_token,\n    jobs.maybe_raw_inference_text,\n\n    jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n    jobs.on_success_result_entity_type as maybe_result_entity_type,\n    jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n    tts_models.title as maybe_tts_model_title,\n    voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n    tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n    voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n    media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n    media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n    media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n    jobs.assigned_worker as maybe_assigned_worker,\n    jobs.assigned_cluster as maybe_assigned_cluster,\n\n    jobs.is_keepalive_required,\n\n    jobs.created_at,\n    jobs.updated_at,\n\n    jobs.first_started_at as maybe_first_started_at,\n    jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n        ",
+  "query": "\nSELECT\n    jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n    jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n    jobs.attempt_count,\n\n    jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n    jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n    jobs.creator_ip_address,\n\n    jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n    jobs.maybe_model_type,\n    jobs.maybe_model_token,\n    jobs.maybe_raw_inference_text,\n\n    jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n    jobs.on_success_result_entity_type as maybe_result_entity_type,\n    jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n    model_weights.title as maybe_model_weights_title,\n    tts_models.title as maybe_tts_model_title,\n    voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n    tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n    voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n    media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n    media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n    media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n    jobs.assigned_worker as maybe_assigned_worker,\n    jobs.assigned_cluster as maybe_assigned_cluster,\n\n    jobs.is_keepalive_required,\n\n    jobs.created_at,\n    jobs.updated_at,\n\n    jobs.first_started_at as maybe_first_started_at,\n    jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n        ",
  "describe": {
    "columns": [
      {
@@ -135,7 +135,7 @@
      },
      {
        "ordinal": 13,
-        "name": "maybe_tts_model_title",
+        "name": "maybe_model_weights_title",
        "type_info": {
          "type": "VarString",
          "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -145,7 +145,7 @@
      },
      {
        "ordinal": 14,
-        "name": "maybe_voice_conversion_model_title",
+        "name": "maybe_tts_model_title",
        "type_info": {
          "type": "VarString",
          "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -155,7 +155,7 @@
      },
      {
        "ordinal": 15,
-        "name": "maybe_tts_public_bucket_path",
+        "name": "maybe_voice_conversion_model_title",
        "type_info": {
          "type": "VarString",
          "flags": "BINARY | NO_DEFAULT_VALUE",
@@ -165,6 +165,16 @@
      },
      {
        "ordinal": 16,
+        "name": "maybe_tts_public_bucket_path",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 17,
        "name": "maybe_voice_conversion_public_bucket_hash",
        "type_info": {
          "type": "VarString",
@@ -174,7 +184,7 @@
        }
      },
      {
-        "ordinal": 17,
+        "ordinal": 18,
        "name": "maybe_media_file_public_bucket_directory_hash",
        "type_info": {
          "type": "VarString",
@@ -184,7 +194,7 @@
        }
      },
      {
-        "ordinal": 18,
+        "ordinal": 19,
        "name": "maybe_media_file_public_bucket_prefix",
        "type_info": {
          "type": "VarString",
@@ -194,7 +204,7 @@
        }
      },
      {
-        "ordinal": 19,
+        "ordinal": 20,
        "name": "maybe_media_file_public_bucket_extension",
        "type_info": {
          "type": "VarString",
@@ -204,7 +214,7 @@
        }
      },
      {
-        "ordinal": 20,
+        "ordinal": 21,
        "name": "maybe_assigned_worker",
        "type_info": {
          "type": "VarString",
@@ -214,7 +224,7 @@
        }
      },
      {
-        "ordinal": 21,
+        "ordinal": 22,
        "name": "maybe_assigned_cluster",
        "type_info": {
          "type": "VarString",
@@ -224,7 +234,7 @@
        }
      },
      {
-        "ordinal": 22,
+        "ordinal": 23,
        "name": "is_keepalive_required",
        "type_info": {
          "type": "Tiny",
@@ -234,7 +244,7 @@
        }
      },
      {
-        "ordinal": 23,
+        "ordinal": 24,
        "name": "created_at",
        "type_info": {
          "type": "Timestamp",
@@ -244,7 +254,7 @@
        }
      },
      {
-        "ordinal": 24,
+        "ordinal": 25,
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
@@ -254,7 +264,7 @@
        }
      },
      {
-        "ordinal": 25,
+        "ordinal": 26,
        "name": "maybe_first_started_at",
        "type_info": {
          "type": "Timestamp",
@@ -264,7 +274,7 @@
        }
      },
      {
-        "ordinal": 26,
+        "ordinal": 27,
        "name": "maybe_successfully_completed_at",
        "type_info": {
          "type": "Timestamp",
@@ -300,6 +310,7 @@
      true,
      true,
      true,
+      true,
      false,
      false,
      false,
@@ -307,5 +318,5 @@
      true
    ]
  },
-  "hash": "4932e975b3369b06fcc23f7797f3a53a54211236383385b8740959b824bd9744"
+  "hash": "5fcdce6dd4c2b24d9062fda7b0a080bda3f04da6933b4dba8d335f9b7aecacb3"
 }
@@ -228,7 +228,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -238,7 +238,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -248,7 +248,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -157,7 +157,7 @@
        "ordinal": 15,
        "name": "file_size_bytes",
        "type_info": {
-          "type": "Long",
+          "type": "LongLong",
          "flags": "NOT_NULL",
          "char_set": 63,
          "max_size": 10
@@ -388,7 +388,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -398,7 +398,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -408,7 +408,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO media_files\nSET\n  token = ?,\n\n  origin_category = ?,\n  origin_product_category = ?,\n\n  maybe_origin_model_type = ?,\n  maybe_origin_model_token = ?,\n\n  maybe_text_transcript = ?,\n\n  media_type = ?,\n  maybe_mime_type = ?,\n  file_size_bytes = ?,\n\n  checksum_sha2 = ?,\n\n  public_bucket_directory_hash = ?,\n  maybe_public_bucket_prefix = ?,\n  maybe_public_bucket_extension = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  maybe_creator_file_synthetic_id = ?,\n  maybe_creator_category_synthetic_id = ?,\n\n  is_generated_on_prem = ?,\n  generated_by_worker = ?,\n  generated_by_cluster = ?\n\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 22
+    },
+    "nullable": []
+  },
+  "hash": "8aad4a39b2e43743a2efc1b37c17860c15a2314145c61ab4a25e449a4cb22128"
+}
@@ -157,7 +157,7 @@
        "ordinal": 15,
        "name": "file_size_bytes",
        "type_info": {
-          "type": "Long",
+          "type": "LongLong",
          "flags": "NOT_NULL",
          "char_set": 63,
          "max_size": 10
@@ -328,7 +328,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -338,7 +338,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -348,7 +348,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -157,7 +157,7 @@
        "ordinal": 15,
        "name": "file_size_bytes",
        "type_info": {
-          "type": "Long",
+          "type": "LongLong",
          "flags": "NOT_NULL",
          "char_set": 63,
          "max_size": 10
@@ -328,7 +328,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -338,7 +338,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -348,7 +348,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -88,7 +88,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -98,7 +98,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -108,7 +108,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -58,7 +58,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -68,7 +68,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "INSERT INTO idempotency_tokens SET idempotency_value = ?",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": []
+  },
+  "hash": "cfeb095db83dfad895835d509818ecccc6cb9da2b32b5155e5ca29d56d3b7fc0"
+}
@@ -58,7 +58,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -68,7 +68,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -157,7 +157,7 @@
        "ordinal": 15,
        "name": "file_size_bytes",
        "type_info": {
-          "type": "Long",
+          "type": "LongLong",
          "flags": "NOT_NULL",
          "char_set": 63,
          "max_size": 10
@@ -288,7 +288,7 @@
        "name": "updated_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "flags": "NOT_NULL | MULTIPLE_KEY | BINARY | TIMESTAMP | ON_UPDATE_NOW",
          "char_set": 63,
          "max_size": 19
        }
@@ -298,7 +298,7 @@
        "name": "user_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -308,7 +308,7 @@
        "name": "mod_deleted_at",
        "type_info": {
          "type": "Timestamp",
-          "flags": "BINARY",
+          "flags": "MULTIPLE_KEY | BINARY",
          "char_set": 63,
          "max_size": 19
        }
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\n    INSERT INTO media_files\n    SET\n      token = ?,\n    \n      origin_category = ?,\n      origin_product_category = ?,\n\n      maybe_origin_model_type = ?,\n\n      maybe_text_transcript = ?,\n\n      media_type = ?,\n      maybe_mime_type = ?,\n      file_size_bytes = ?,\n    \n      checksum_sha2 = ?,\n    \n      public_bucket_directory_hash = ?,\n      maybe_public_bucket_prefix = ?,\n      maybe_public_bucket_extension = ?,\n    \n      maybe_creator_user_token = ?,\n      maybe_creator_anonymous_visitor_token = ?,\n      creator_ip_address = ?,\n    \n      creator_set_visibility = ?,\n    \n      maybe_creator_file_synthetic_id = ?,\n      maybe_creator_category_synthetic_id = ?,\n    \n      is_generated_on_prem = ?,\n      generated_by_worker = ?,\n      generated_by_cluster = ?\n    \n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 21
+    },
+    "nullable": []
+  },
+  "hash": "e7f9bb387905704e9da3fd1657432a3d821d49462fcf82d8cca8dd2dcfb3df17"
+}
@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  job_type = ?,\n\n  inference_category = ?,\n  maybe_model_type = ?,\n  maybe_model_token = ?,\n\n  maybe_input_source_token = ?,\n  maybe_input_source_token_type = ?,\n\n  maybe_raw_inference_text = ?,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  priority_level = ?,\n  is_keepalive_required = ?,\n  max_duration_seconds = ?,\n\n  is_debug_request = ?,\n  maybe_routing_tag = ?,\n\n  status = \"pending\"\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 19
-    },
-    "nullable": []
-  },
-  "hash": "e97a0974ed643b8a20d4d319b4aa93aba979d5f8fc0bdc44b77ef4f945b03b35"
-}
@@ -762,6 +762,51 @@ dependencies = [
 "thiserror",
 ]

+[[package]]
+name = "axum"
+version = "0.6.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bitflags 1.3.2",
+ "bytes 1.5.0",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "itoa 1.0.9",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
+dependencies = [
+ "async-trait",
+ "bytes 1.5.0",
+ "futures-util",
+ "http",
+ "http-body",
+ "mime",
+ "rustversion",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.69"
@@ -2292,6 +2337,50 @@ dependencies = [
 "my-workspace-hack",
 ]

+[[package]]
+name = "es-update-job"
+version = "0.0.1"
+dependencies = [
+ "actix_helpers",
+ "anyhow",
+ "bootstrap",
+ "chrono",
+ "concurrency",
+ "config",
+ "container_common",
+ "dotenv",
+ "easyenv",
+ "elasticsearch",
+ "elasticsearch_schema",
+ "enums",
+ "errors",
+ "filesys",
+ "hashing",
+ "hostname",
+ "jobs_common",
+ "log",
+ "media",
+ "memory_caching",
+ "mimetypes",
+ "my-workspace-hack",
+ "mysql_queries",
+ "once_cell",
+ "rand 0.8.5",
+ "redis_common",
+ "reusable_types",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "server_environment",
+ "sqlx",
+ "storyteller_root",
+ "subprocess",
+ "subprocess_common",
+ "testing",
+ "tokens",
+ "tokio",
+]
+
 [[package]]
 name = "etcetera"
 version = "0.8.0"
@@ -2650,6 +2739,12 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"

+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
 [[package]]
 name = "google_drive_common"
 version = "0.0.1"
@@ -2674,7 +2769,7 @@ dependencies = [
 "futures-sink",
 "futures-util",
 "http",
- "indexmap",
+ "indexmap 2.1.0",
 "slab",
 "tokio",
 "tokio-util",
@@ -2928,6 +3023,18 @@ dependencies = [
 "want",
 ]

+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.5.0"
@@ -3023,6 +3130,16 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"

+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.1.0"
@@ -3096,6 +3213,9 @@ dependencies = [
 "my-workspace-hack",
 "mysql_queries",
 "once_cell",
+ "opentelemetry",
+ "opentelemetry-otlp",
+ "opentelemetry_sdk",
 "r2d2_redis",
 "rand 0.8.5",
 "redis_common",
@@ -3523,6 +3643,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"

+[[package]]
+name = "matchit"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
+
 [[package]]
 name = "maybe-async"
 version = "0.2.7"
@@ -3834,7 +3960,7 @@ dependencies = [
 "heck",
 "hmac 0.12.1",
 "hyper",
- "indexmap",
+ "indexmap 2.1.0",
 "lazy_static",
 "libc",
 "log",
@@ -4197,6 +4323,93 @@ dependencies = [
 "vcpkg",
 ]

+[[package]]
+name = "opentelemetry"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "indexmap 2.1.0",
+ "js-sys",
+ "once_cell",
+ "pin-project-lite",
+ "thiserror",
+ "urlencoding",
+]
+
+[[package]]
+name = "opentelemetry-otlp"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930"
+dependencies = [
+ "async-trait",
+ "futures-core",
+ "http",
+ "opentelemetry",
+ "opentelemetry-proto",
+ "opentelemetry-semantic-conventions",
+ "opentelemetry_sdk",
+ "prost",
+ "thiserror",
+ "tokio",
+ "tonic",
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1"
+dependencies = [
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "prost",
+ "tonic",
+]
+
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84"
+dependencies = [
+ "opentelemetry",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4"
+dependencies = [
+ "async-trait",
+ "crossbeam-channel",
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "glob",
+ "once_cell",
+ "opentelemetry",
+ "ordered-float",
+ "percent-encoding",
+ "rand 0.8.5",
+ "thiserror",
+ "tokio",
+ "tokio-stream",
+]
+
+[[package]]
+name = "ordered-float"
+version = "4.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "ordered-multimap"
 version = "0.4.3"
@@ -4554,6 +4767,29 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "prost"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
+dependencies = [
+ "bytes 1.5.0",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
+dependencies = [
+ "anyhow",
+ "itertools 0.10.5",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "psm"
 version = "0.1.21"
@@ -5404,7 +5640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "224e6a14f315852940f3ec103125aa6482f0e224732ed91ed3330ed633077c34"
 dependencies = [
 "form_urlencoded",
- "indexmap",
+ "indexmap 2.1.0",
 "itoa 1.0.9",
 "ryu",
 "serde",
@@ -5416,7 +5652,7 @@ version = "1.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
 "itoa 1.0.9",
 "ryu",
 "serde",
@@ -5842,7 +6078,7 @@ dependencies = [
 "futures-util",
 "hashlink",
 "hex",
- "indexmap",
+ "indexmap 2.1.0",
 "log",
 "memchr",
 "native-tls",
@@ -6441,6 +6677,12 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
 [[package]]
 name = "system-configuration"
 version = "0.5.1"
@@ -6684,6 +6926,16 @@ dependencies = [
 "log",
 ]

+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-macros"
 version = "2.2.0"
@@ -6855,13 +7107,67 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
 "serde",
 "serde_spanned",
 "toml_datetime",
 "winnow",
 ]

+[[package]]
+name = "tonic"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
+dependencies = [
+ "async-trait",
+ "axum",
+ "base64 0.21.5",
+ "bytes 1.5.0",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap 1.9.3",
+ "pin-project",
+ "pin-project-lite",
+ "rand 0.8.5",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+
 [[package]]
 name = "tower-service"
 version = "0.3.2"
@@ -7298,6 +7604,8 @@ dependencies = [
 "toml 0.5.11",
 "user_input_common",
 "user_traits_component",
+ "utoipa",
+ "utoipa-swagger-ui",
 ]

 [[package]]
@@ -7318,7 +7626,7 @@ version = "4.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
 "serde",
 "serde_json",
 "utoipa-gen",
@@ -73,6 +73,7 @@ members = [
  "crates/service/job/analytics_job",
  "crates/service/job/download_job",
  "crates/service/job/email_sender_job",
+  "crates/service/job/es-update-job",
  "crates/service/job/inference_job",
  "crates/service/job/tts_download_job",
  "crates/service/job/tts_inference_job",
@@ -34,7 +34,13 @@
      "created_at" : { "type" : "date" },
      "updated_at" : { "type" : "date" },
      "user_deleted_at" : { "type" : "date" },
-      "mod_deleted_at" : { "type" : "date" }
+      "mod_deleted_at" : { "type" : "date" },
+
+      "is_deleted" : {
+        "type": "boolean",
+        "index": true
+      }
+
    }
  }
 }
@@ -0,0 +1,28 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- Staff
+-- teddanson is victor's alt
+-- SELECT token, username FROM users WHERE username IN (
+UPDATE users SET can_access_studio = true WHERE username IN (
+    'bflat',
+    'brandon',
+    'crossproduct',
+    'crossproduct1',
+    'echelon',
+    'fyscott',
+    'olivicmic',
+    'saltacc',
+    'teddanson',
+    'vegito1089',
+    'wilwong',
+    'yae_ph',
+    'zzz_last_item'
+);
+
+UPDATE users SET can_access_studio = true WHERE username IN (
+    'endtimes',
+    'tropicalfun',
+    'zzz_last_item'
+);
@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+DROP INDEX index_updated_at ON model_weights;
+DROP INDEX index_user_deleted_at ON model_weights;
+DROP INDEX index_mod_deleted_at ON model_weights;
@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+CREATE INDEX index_updated_at ON model_weights (updated_at);
+CREATE INDEX index_user_deleted_at ON model_weights (user_deleted_at);
+CREATE INDEX index_mod_deleted_at ON model_weights (mod_deleted_at);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users DROP COLUMN can_access_studio;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE users
+    ADD COLUMN can_access_studio BOOLEAN NOT NULL DEFAULT false
+    AFTER user_role_slug;
+
+ALTER TABLE users
+    ADD INDEX index_can_access_studio (can_access_studio);
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    MODIFY COLUMN file_size_bytes INT(10) NOT NULL DEFAULT 0;
@@ -0,0 +1,6 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE model_weights
+    MODIFY COLUMN file_size_bytes BIGINT(10) NOT NULL DEFAULT 0;
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN maybe_text_transcript;
@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN maybe_text_transcript TEXT DEFAULT NULL
+    AFTER maybe_frame_height;
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files DROP COLUMN maybe_batch_token;
@@ -0,0 +1,10 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE media_files
+    ADD COLUMN maybe_batch_token VARCHAR(32) DEFAULT NULL
+    AFTER is_batch_generated;
+
+ALTER TABLE media_files
+    ADD INDEX index_maybe_batch_token (maybe_batch_token);
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN maybe_download_url;
@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN maybe_download_url VARCHAR(1024) DEFAULT NULL
+    AFTER maybe_raw_inference_text;
@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs DROP COLUMN on_success_result_batch_token;
@@ -0,0 +1,7 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+ALTER TABLE generic_inference_jobs
+    ADD COLUMN on_success_result_batch_token VARCHAR(32) DEFAULT NULL
+    AFTER on_success_result_entity_token;
@@ -72,6 +72,11 @@ CREATE TABLE generic_inference_jobs (
  -- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
  maybe_raw_inference_text TEXT DEFAULT NULL,

+  -- For download-related jobs, this is the URL
+  -- This is nullable because not all inference jobs have a download URL.
+  -- The length of 1024 is arbitrary, but we shouldn't need anything longer.
+  maybe_download_url VARCHAR(1024) DEFAULT NULL,
+
  -- ========== SUCCESS CASE ==========

  -- The type of the object will vary based on the type of the upload,
@@ -82,6 +87,11 @@ CREATE TABLE generic_inference_jobs (
  -- This is only populated on a successful result.
  on_success_result_entity_token VARCHAR(32) DEFAULT NULL,

+  -- If the job completes successfully, this is the batch token (ie. when
+  -- several results are created).
+  -- Only populated if there are actual batches generated.
+  on_success_result_batch_token VARCHAR(32) DEFAULT NULL,
+
  -- ========== CREATOR DETAILS AND PREFERENCES ==========

  -- Foreign key to user
@@ -58,11 +58,15 @@ CREATE TABLE media_files (
  -- The original filename of the media (if uploaded by a user)
  maybe_origin_filename VARCHAR(255) DEFAULT NULL,

+  -- TODO: Remove after `maybe_batch_token` gains use.
  -- Whether this media file was generated as part of a batch
  -- If so, we can look up the batch in a separate query/call to the `batch_generations` table.
  -- We won't hold the batch token in this table since it'll be very sparsely populated.
  is_batch_generated BOOLEAN NOT NULL DEFAULT FALSE,

+  -- If the media file is generated as part of a batch, this designates the batch.
+  maybe_batch_token VARCHAR(32) DEFAULT NULL,
+
  -- ========== METADATA ==========

  -- We can store text transcripts, etc. in other tables and refer back via foreign key.
@@ -100,6 +104,9 @@ CREATE TABLE media_files (
  -- For videos, the original frame height.
  maybe_frame_height INT(5) DEFAULT NULL,

+  -- Optional text transcript for audio or video (especially TTS)
+  maybe_text_transcript TEXT DEFAULT NULL,
+
  -- Checksum of the original media
  -- SHA1 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
  -- checksum_sha1 CHAR(40) NOT NULL,
@@ -212,6 +219,7 @@ CREATE TABLE media_files (
  KEY index_maybe_origin_model_type (maybe_origin_model_type),
  KEY fk_maybe_origin_model_token (maybe_origin_model_token),
  KEY fk_maybe_origin_model_type_and_token (maybe_origin_model_type, maybe_origin_model_token),
+  KEY index_maybe_batch_token (maybe_batch_token),
  KEY index_media_type (media_type),
  KEY index_checksum_sha2 (checksum_sha2),
  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
@@ -95,8 +95,8 @@ CREATE TABLE model_weights (
  -- The filename that was used at upload time (if available)
  original_filename VARCHAR(255) DEFAULT NULL,

-  -- File characteristics
-  file_size_bytes INT(10) NOT NULL DEFAULT 0,
+  -- File size. BIGINT to store more than 2GB size.
+  file_size_bytes BIGINT(10) NOT NULL DEFAULT 0,

  -- Checksum of the original weights file
  -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
@@ -171,6 +171,9 @@ CREATE TABLE model_weights (
  KEY index_weights_category (weights_category),
  KEY fk_creator_user_token (creator_user_token),
  KEY index_creator_set_visibility (creator_set_visibility),
-  KEY fk_maybe_cover_media_file_token (maybe_cover_image_media_file_token)
+  KEY fk_maybe_cover_media_file_token (maybe_cover_image_media_file_token),
+  KEY index_updated_at (updated_at),
+  KEY index_user_deleted_at (user_deleted_at),
+  KEY index_mod_deleted_at (user_deleted_at)

 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
@@ -22,6 +22,10 @@ CREATE TABLE users (
  -- The role assigned to the user confers permissions.
  user_role_slug VARCHAR(16) NOT NULL,

+  -- Rollout flag for Storyteller Studio
+  -- Default to false until rollout is 100%, then remove the column.
+  can_access_studio BOOLEAN NOT NULL DEFAULT false,
+
  -- ========== PREMIUM FEATURES ==========

  -- If the user has a Stripe subscription (or has had one), we link it to the user here.
@@ -171,6 +175,7 @@ CREATE TABLE users (
  UNIQUE KEY (username),
  UNIQUE KEY (email_address),
  KEY fk_user_role_slug (user_role_slug),
+  KEY index_can_access_studio (can_access_studio),
  KEY fk_maybe_avatar_media_file_token (maybe_avatar_media_file_token),
  KEY fk_maybe_cover_media_file_token (maybe_cover_media_file_token)

@@ -40,6 +40,9 @@ pub enum InferenceCategory {

  #[serde(rename = "format_conversion")]
  FormatConversion,
+
+  #[serde(rename = "convert_bvh_to_workflow")]
+  ConvertBvhToWorkflow,
 }

 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -58,6 +61,7 @@ impl InferenceCategory {
      Self::Mocap => "mocap",
      Self::Workflow => "workflow",
      Self::FormatConversion => "format_conversion",
+      Self::ConvertBvhToWorkflow => "convert_bvh_to_workflow",
    }
  }

@@ -71,6 +75,7 @@ impl InferenceCategory {
      "mocap" => Ok(Self::Mocap),
      "workflow" => Ok(Self::Workflow),
      "format_conversion" => Ok(Self::FormatConversion),
+      "convert_bvh_to_workflow" => Ok(Self::ConvertBvhToWorkflow),
      _ => Err(format!("invalid value: {:?}", value)),
    }
  }
@@ -87,6 +92,7 @@ impl InferenceCategory {
      Self::Mocap,
      Self::Workflow,
      Self::FormatConversion,
+      Self::ConvertBvhToWorkflow,
    ])
  }
 }
@@ -109,6 +115,7 @@ mod tests {
      assert_serialization(InferenceCategory::Mocap, "mocap");
      assert_serialization(InferenceCategory::Workflow, "workflow");
      assert_serialization(InferenceCategory::FormatConversion, "format_conversion");
+      assert_serialization(InferenceCategory::ConvertBvhToWorkflow, "convert_bvh_to_workflow");
    }

    #[test]
@@ -121,6 +128,7 @@ mod tests {
      assert_eq!(InferenceCategory::Mocap.to_str(), "mocap");
      assert_eq!(InferenceCategory::Workflow.to_str(), "workflow");
      assert_eq!(InferenceCategory::FormatConversion.to_str(), "format_conversion");
+      assert_eq!(InferenceCategory::ConvertBvhToWorkflow.to_str(), "convert_bvh_to_workflow");
    }

    #[test]
@@ -133,6 +141,7 @@ mod tests {
      assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap);
      assert_eq!(InferenceCategory::from_str("workflow").unwrap(), InferenceCategory::Workflow);
      assert_eq!(InferenceCategory::from_str("format_conversion").unwrap(), InferenceCategory::FormatConversion);
+      assert_eq!(InferenceCategory::from_str("convert_bvh_to_workflow").unwrap(), InferenceCategory::ConvertBvhToWorkflow);
    }

    #[test]
@@ -148,6 +157,7 @@ mod tests {
      assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap));
      assert_eq!(variants.pop_first(), Some(InferenceCategory::Workflow));
      assert_eq!(variants.pop_first(), Some(InferenceCategory::FormatConversion));
+      assert_eq!(variants.pop_first(), Some(InferenceCategory::ConvertBvhToWorkflow));
      assert_eq!(variants.pop_first(), None);

      // Generated check
@@ -15,6 +15,10 @@ use strum::EnumIter;
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)]
 #[serde(rename_all = "snake_case")]
 pub enum InferenceJobType {
+  /// Job that converts bevy to workflow files
+  #[serde(rename = "bevy_to_workflow")]
+  BevyToWorkflow,
+
  /// Jobs that run ComfyUI workflows
  ComfyUi,

@@ -64,6 +68,7 @@ impl_mysql_enum_coders!(InferenceJobType);
 impl InferenceJobType {
  pub fn to_str(&self) -> &'static str {
    match self {
+      Self::BevyToWorkflow => "bevy_to_workflow",
      Self::ComfyUi => "comfy_ui",
      Self::ConvertFbxToGltf => "convert_fbx_gltf",
      Self::MocapNet => "mocap_net",
@@ -80,6 +85,7 @@ impl InferenceJobType {

  pub fn from_str(value: &str) -> Result<Self, String> {
    match value {
+      "bevy_to_workflow" => Ok(Self::BevyToWorkflow),
      "comfy_ui" => Ok(Self::ComfyUi),
      "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
      "mocap_net" => Ok(Self::MocapNet),
@@ -99,6 +105,7 @@ impl InferenceJobType {
    // NB: BTreeSet is sorted
    // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
    BTreeSet::from([
+      Self::BevyToWorkflow,
      Self::ComfyUi,
      Self::ConvertFbxToGltf,
      Self::MocapNet,
@@ -129,6 +136,7 @@ mod tests {

    #[test]
    fn test_serialization() {
+      assert_serialization(InferenceJobType::BevyToWorkflow, "bevy_to_workflow");
      assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
      assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
      assert_serialization(InferenceJobType::MocapNet, "mocap_net");
@@ -144,6 +152,7 @@ mod tests {

    #[test]
    fn to_str() {
+      assert_eq!(InferenceJobType::BevyToWorkflow.to_str(), "bevy_to_workflow");
      assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
      assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
      assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
@@ -159,6 +168,7 @@ mod tests {

    #[test]
    fn from_str() {
+      assert_eq!(InferenceJobType::from_str("bevy_to_workflow").unwrap(), InferenceJobType::BevyToWorkflow);
      assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
      assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
      assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
@@ -176,7 +186,8 @@ mod tests {
    fn all_variants() {
      // Static check
      let mut variants = InferenceJobType::all_variants();
-      assert_eq!(variants.len(), 11);
+      assert_eq!(variants.len(), 12);
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::BevyToWorkflow));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
      assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
@@ -44,6 +44,8 @@ pub enum InferenceModelType {
  /// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
  #[serde(rename = "convert_fbx_gltf")]
  ConvertFbxToGltf,
+  #[serde(rename = "bvh_to_workflow")]
+  BvhToWorkflow
 }

 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -66,6 +68,7 @@ impl InferenceModelType {
      Self::StyleTTS2 => "styletts2",
      Self::ComfyUi => "comfy_ui",
      Self::ConvertFbxToGltf => "convert_fbx_gltf",
+      Self::BvhToWorkflow => "bvh_to_workflow",
    }
  }

@@ -83,6 +86,7 @@ impl InferenceModelType {
      "styletts2" => Ok(Self::StyleTTS2),
      "comfy_ui" => Ok(Self::ComfyUi),
      "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
+      "bvh_to_workflow" => Ok(Self::BvhToWorkflow),
      _ => Err(format!("invalid value: {:?}", value)),
    }
  }
@@ -103,6 +107,7 @@ impl InferenceModelType {
      Self::StyleTTS2,
      Self::ComfyUi,
      Self::ConvertFbxToGltf,
+        Self::BvhToWorkflow,
    ])
  }
 }
@@ -129,6 +134,7 @@ mod tests {
      assert_serialization(InferenceModelType::ComfyUi, "comfy_ui");
      assert_serialization(InferenceModelType::StyleTTS2, "styletts2");
      assert_serialization(InferenceModelType::ConvertFbxToGltf, "convert_fbx_gltf");
+      assert_serialization(InferenceModelType::BvhToWorkflow, "bvh_to_workflow");
    }

    #[test]
@@ -145,6 +151,7 @@ mod tests {
      assert_eq!(InferenceModelType::StyleTTS2.to_str(), "styletts2");
      assert_eq!(InferenceModelType::ComfyUi.to_str(), "comfy_ui");
      assert_eq!(InferenceModelType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
+      assert_eq!(InferenceModelType::BvhToWorkflow.to_str(), "bvh_to_workflow");
    }

    #[test]
@@ -161,6 +168,7 @@ mod tests {
      assert_eq!(InferenceModelType::from_str("styletts2").unwrap(), InferenceModelType::StyleTTS2);
      assert_eq!(InferenceModelType::from_str("comfy_ui").unwrap(), InferenceModelType::ComfyUi);
      assert_eq!(InferenceModelType::from_str("convert_fbx_gltf").unwrap(), InferenceModelType::ConvertFbxToGltf);
+      assert_eq!(InferenceModelType::from_str("bvh_to_workflow").unwrap(), InferenceModelType::BvhToWorkflow);
    }

    #[test]
@@ -180,6 +188,7 @@ mod tests {
      assert_eq!(variants.pop_first(), Some(InferenceModelType::ComfyUi));
      assert_eq!(variants.pop_first(), Some(InferenceModelType::StyleTTS2));
      assert_eq!(variants.pop_first(), Some(InferenceModelType::ConvertFbxToGltf));
+      assert_eq!(variants.pop_first(), Some(InferenceModelType::BvhToWorkflow));
      assert_eq!(variants.pop_first(), None);

      // Generated check
@@ -2,11 +2,12 @@ use std::fmt::Debug;

 use serde::Deserialize;
 use serde::Serialize;
+use utoipa::ToSchema;

 use crate::prefixes::TokenPrefix;

 /// The primary key for the  "zs_voice_datasets" table.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)]
 #[sqlx(transparent)]
 pub struct ZsVoiceDatasetToken(pub String);

@@ -8,7 +8,7 @@ use errors::AnyhowResult;
 use crate::bucket_clients::get_bucket_clients;
 use crate::cli_args::parse_cli_args;
 use crate::seeding::model_weights::seed_weights;
-
+use crate::seeding::users::seed_user_accounts;
 pub mod bucket_clients;

 pub mod cli_args;
@@ -47,11 +47,11 @@ pub async fn main() -> AnyhowResult<()> {
 //    maybe_elasticsearch = Some(get_elasticsearch_client());
 //  }

-  // seed_user_accounts(&pool).await?;
+  //seed_user_accounts(&pool).await?;
  // seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?;
  //seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?;
  // seed_voice_conversion(&pool).await?;
-  seed_weights(&pool).await?;
+  //seed_weights(&pool).await?;
  // seed_media_seedtool(&pool).await?;
  // seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?;
  
@@ -41,14 +41,14 @@ pub async fn test_seed_weights_files() -> AnyhowResult<()> {

    let bucket_details1 = RemoteCloudBucketDetails {
        object_hash: String::from("apa0ej6es8d3ss2gwtf1cghge35qn9tn"),
-        prefix: String::from("sd15"),
-        suffix: String::from("safetensors"),
+        prefix: String::from("sd15_"),
+        suffix: String::from(".safetensors"),
    };

    let bucket_details2 = RemoteCloudBucketDetails {
        object_hash: String::from("27kz11et18fargyyxbj66ntfn621k9d3"),
-        prefix: String::from("loRA"),
-        suffix: String::from("safetensors"),
+        prefix: String::from("loRA_"),
+        suffix: String::from(".safetensors"),
    };

    remote_cloud_file_client.download_file(bucket_details1, String::from("./checkpoint")).await?;
@@ -331,10 +331,6 @@ pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserT
                // private_bucket_hash = format!("bucket_hash{}", i);
                private_bucket_prefix = format!("_fake");
                private_bucket_extension = format!("rvcV2");
-                //cached_user_ratings_total_count = i;
-                //cached_user_ratings_positive_count = i;
-                //cached_user_ratings_negative_count = i;
-                //cached_user_ratings_ratio = i as u32 / 100;
                version = i as i32;

                println!("Out of range");
@@ -836,8 +832,8 @@ pub async fn seed_weights_for_user_token(
            file_size_bytes: 1024,
            file_checksum_sha2: "checksum1".to_string(),
            public_bucket_hash: "bucket_hash1".to_string(),
-            maybe_public_bucket_prefix: Some("_fake".to_string()),
-            maybe_public_bucket_extension: Some("rvc".to_string()),
+            maybe_public_bucket_prefix: Some("rvc_".to_string()),
+            maybe_public_bucket_extension: Some(".pt".to_string()),
            //cached_user_ratings_total_count: 10,
            //cached_user_ratings_positive_count: 9,
            //cached_user_ratings_negative_count: 1,
@@ -1119,9 +1115,9 @@ pub async fn seed_weights(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
    // original_seed_weights(mysql_pool,user_token).await?;
    // seed_weights_for_user_token(mysql_pool, user_token).await?;
    // seed_weights_for_paging(mysql_pool,user_token).await?;
-    // seed_weights_for_testing_inference(mysql_pool,user_token.clone()).await?;
+    seed_weights_for_testing_inference(mysql_pool,user_token.clone()).await?;
    // println!("TESTING DOWLOAD");
-    // test_seed_weights_files().await?;
-    seed_workflows_for_testing_inference(mysql_pool,user_token.clone()).await?;
+    //test_seed_weights_files().await?;
+    // seed_workflows_for_testing_inference(mysql_pool,user_token.clone()).await?;
    Ok(())
 }
@@ -33,7 +33,7 @@ pub enum Action {
 #[command(name="elasticsearch-cli")]
 pub struct Args {
  #[arg(name="action", long="action", help="action to take", required=true)]
-  action: String,
+  action: Action,

  #[arg(name="mysql", long="mysql", help="production or development")]
  mysql: Option<String>,
@@ -48,7 +48,7 @@ pub fn parse_cli_args() -> AnyhowResult<ParsedArgs> {
  Ok(ParsedArgs {
    mysql_environment: to_environment(args.mysql.as_deref())?,
    elasticsearch_environment: to_environment(args.elasticsearch.as_deref())?,
-    action: action_from_str(&args.action)?,
+    action: args.action,
  })
 }

@@ -53,6 +53,8 @@ pub async fn create_all_model_weight_documents(mysql: &Pool<MySql>, elasticsearc
 async fn create_document_from_record(elasticsearch: &Elasticsearch, record: ModelWeightForElasticsearchRecord) -> AnyhowResult<()> {
  info!("Create record for {:?} - {:?}", record.token, record.title);

+  let is_deleted = record.user_deleted_at.is_some() || record.mod_deleted_at.is_some();
+
  let document = ModelWeightDocument {
    token: record.token,

@@ -93,6 +95,8 @@ async fn create_document_from_record(elasticsearch: &Elasticsearch, record: Mode
    updated_at: record.updated_at,
    user_deleted_at: record.user_deleted_at,
    mod_deleted_at: record.mod_deleted_at,
+
+    is_deleted,
  };

  let op : BulkOperation<_> = BulkOperation::index(&document)
@@ -26,7 +26,7 @@ pub struct FileBucketDirectory {
      // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
      let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
      // gets you name of the file with suffix and prefix and entropy in the centre
-      let file_name = format!("{}_{}.{}", bucket_details.get_prefix(), entropy, bucket_details.get_suffix());
+      let file_name = format!("{}{}{}", bucket_details.get_prefix(), entropy, bucket_details.get_suffix());
      // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
      // This is the full path you download from
      let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name);
@@ -46,7 +46,7 @@ pub struct FileBucketDirectory {
      // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
      let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
      // gets you name of the file with suffix and prefix and entropy in the centre
-      let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix());
+      let file_name = format!("{}{}{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix());
      // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.

      // This is the full path you upload to.
@@ -1,4 +1,5 @@
 use crate::remote_file_manager::file_descriptor::FileDescriptor;
+
 // TODO ASK BRANDON WHAT path should this be?
 const MEDIA_FILE_DIRECTORY: &str = "/media";

@@ -9,11 +10,11 @@ impl FileDescriptor for MediaImagePngDescriptor {
        MEDIA_FILE_DIRECTORY
    }
    fn get_suffix(&self)->String {
-        "png".to_string()
+        ".png".to_string()
    }

    fn get_prefix(&self)->String {
-        "image".to_string()
+        "image_".to_string()
    }

    fn is_public(&self) -> bool {
@@ -27,11 +28,29 @@ impl FileDescriptor for MediaVideoMp4Descriptor {
        MEDIA_FILE_DIRECTORY
    }
    fn get_suffix(&self)->String {
-        "mp4".to_string()
+        ".mp4".to_string()
    }

    fn get_prefix(&self)->String {
-        "video".to_string()
+        "video_".to_string()
+    }
+
+    fn is_public(&self) -> bool {
+        true
+    }
+}
+
+pub struct UploadVideoMp4Descriptor;
+impl FileDescriptor for UploadVideoMp4Descriptor {
+    fn remote_directory_path(&self) -> &str {
+        MEDIA_FILE_DIRECTORY
+    }
+    fn get_suffix(&self)->String {
+        ".mp4".to_string()
+    }
+
+    fn get_prefix(&self)->String {
+        "upload_".to_string()
    }

    fn is_public(&self) -> bool {
@@ -27,37 +27,44 @@ impl RemoteCloudBucketDetails {
        &self.suffix
    }

+    // this is not really useful for downloads remove...
    pub fn file_descriptor_from_bucket_details(&self) -> Box<dyn FileDescriptor> {
        match self.prefix.as_str() {
            // Weights
-            "loRA" => Box::new(weights_descriptor::WeightsLoRADescriptor {}),
-            "sd15" =>
+            "loRA_" => Box::new(weights_descriptor::WeightsLoRADescriptor {}),
+            "sd15_" =>
                match self.suffix.as_str() {
-                    "safetensors" => Box::new(weights_descriptor::WeightsSD15Descriptor {}),
-                    "ckpt" => Box::new(weights_descriptor::WeightsSD15CkptDescriptor {}),
+                    ".safetensors" => Box::new(weights_descriptor::WeightsSD15Descriptor {}),
+                    ".ckpt" => Box::new(weights_descriptor::WeightsSD15CkptDescriptor {}),
                    _ => panic!("Unknown suffix: {}",self.suffix)
                },
-            "sdxl" => Box::new(weights_descriptor::WeightsSDXLDescriptor {}),
-            "valle_prompt" => Box::new(weights_descriptor::WeightsVallePromptDescriptor {}),
-            "rvc" => {
+            "sdxl_" => Box::new(weights_descriptor::WeightsSDXLDescriptor {}),
+            "valle_prompt_" => Box::new(weights_descriptor::WeightsVallePromptDescriptor {}),
+            "rvc_" => {
                match self.suffix.as_str() {
-                    "safetensors" => Box::new(weights_descriptor::WeightsRVCDescriptor {}),
-                    "index" => Box::new(weights_descriptor::WeightsRVCIndexDescriptor {}),
+                    ".safetensors" => Box::new(weights_descriptor::WeightsRVCDescriptor {}),
+                    ".index" => Box::new(weights_descriptor::WeightsRVCIndexDescriptor {}),
                    _ => panic!("Unknown suffix: {}",self.suffix)
                }
            },
-            "svc" => Box::new(weights_descriptor::WeightsSVCDescriptor {}),
-            "workflow" => Box::new(weights_descriptor::WeightsWorkflowDescriptor {}),
+            "svc_" => Box::new(weights_descriptor::WeightsSVCDescriptor {}),
+            "workflow_" => Box::new(weights_descriptor::WeightsWorkflowDescriptor {}),
            // Media
-            "image" => {
+            "image_" => {
                match self.suffix.as_str() {
-                    "png" => Box::new(media_descriptor::MediaImagePngDescriptor {}),
+                    ".png" => Box::new(media_descriptor::MediaImagePngDescriptor {}),
                    _ => panic!("Unknown suffix: {}",self.suffix)
                }
            },
-            "video" => {
+            "video_" => {
                match self.suffix.as_str() {
-                    "mp4" => Box::new(media_descriptor::MediaVideoMp4Descriptor {}),
+                    ".mp4" => Box::new(media_descriptor::MediaVideoMp4Descriptor {}),
+                    _ => panic!("Unknown suffix: {}",self.suffix)
+                }
+            },
+            "upload_" => {
+                match self.suffix.as_str() {
+                    ".mp4" => Box::new(media_descriptor::UploadVideoMp4Descriptor {}),
                    _ => panic!("Unknown suffix: {}",self.suffix)
                }
            },
@@ -1,4 +1,5 @@
 use anyhow::anyhow;
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;

 use errors::AnyhowResult;
 use filesys::file_read_bytes::file_read_bytes;
@@ -49,6 +50,14 @@ impl RemoteCloudFileClient {
        Ok(())
    }

+    pub async fn download_media_file(&self, media_file_path: &MediaFileBucketPath, to_system_file_path: String) -> AnyhowResult<()> {
+        const IS_PUBLIC: bool = true; // NB: Media files are always public
+        let full_remote_cloud_file_path = media_file_path.get_full_object_path_str().to_string();
+
+        self.bucket_orchestration_client.download_file_to_disk(full_remote_cloud_file_path, to_system_file_path, IS_PUBLIC).await?;
+        Ok(())
+    }
+
    // return error or success with meta data.
    pub async fn upload_file(&self, file_descriptor:Box<dyn FileDescriptor>, from_system_file_path:&str) -> AnyhowResult<FileMetaData> {
     
@@ -153,11 +162,11 @@ mod tests {
    #[tokio::test]
    async fn remote_file_manager_descriptor_test() {
        use super::*;
-        let remote_cloud_bucket_details = RemoteCloudBucketDetails::new("object_hash".to_string(), "loRA".to_string(), "safetensors".to_string());
+        let remote_cloud_bucket_details = RemoteCloudBucketDetails::new("object_hash".to_string(), "loRA_".to_string(), ".safetensors".to_string());
        let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details();
        
-        assert_eq!(file_descriptor.get_prefix(), "loRA");
-        assert_eq!(file_descriptor.get_suffix(), "safetensors");
+        assert_eq!(file_descriptor.get_prefix(), "loRA_");
+        assert_eq!(file_descriptor.get_suffix(), ".safetensors");
        assert_eq!(file_descriptor.is_public(), true);

    }
@@ -178,8 +187,8 @@ mod tests {
        println!("begin upload from file_path: {:?}", file_path);
        let details = RemoteCloudBucketDetails {
            object_hash: String::from("123"),
-            prefix: String::from("loRA"),
-            suffix: String::from("safetensors")
+            prefix: String::from("loRA_"),
+            suffix: String::from(".safetensors")
        };
        let result = remote_cloud_file_manager.download_file(details,file_path.to_string()).await;
        
@@ -10,13 +10,13 @@ impl FileDescriptor for WeightsLoRADescriptor {
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-       "safetensors".to_string()
+       ".safetensors".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, loRA, sd15, sdxl when implmenting add to the end 
    fn get_prefix(&self)->String {
-        "loRA".to_string()
+        "loRA_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -35,13 +35,13 @@ impl FileDescriptor for WeightsSD15Descriptor {
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-       "safetensors".to_string()
+       ".safetensors".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end 
    fn get_prefix(&self)->String {
-        "sd15".to_string()
+        "sd15_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -59,13 +59,13 @@ impl FileDescriptor for WeightsSD15CkptDescriptor {
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-       "ckpt".to_string()
+       ".ckpt".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
    fn get_prefix(&self)->String {
-        "sd15".to_string()
+        "sd15_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -84,13 +84,13 @@ impl FileDescriptor for WeightsSDXLDescriptor {
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-       "safetensors".to_string()
+       ".safetensors".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end 
    fn get_prefix(&self)->String {
-        "sdxl".to_string()
+        "sdxl_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -108,13 +108,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsS
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-        "safetensors".to_string()
+        ".safetensors".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
    fn get_prefix(&self)->String {
-        "svc".to_string()
+        "svc_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -132,13 +132,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-        "pth".to_string()
+        ".pth".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
    fn get_prefix(&self)->String {
-        "rvc".to_string()
+        "rvc_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -156,13 +156,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-        "index".to_string()
+        ".index".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
    fn get_prefix(&self)->String {
-        "rvc".to_string()
+        "rvc_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -180,13 +180,13 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsV
    // this will be the type of file peroid is handled by the file formatter
    // e.g safetensors bin jpg
    fn get_suffix(&self)->String {
-        "safetensors".to_string()
+        ".safetensors".to_string()
    }
    // This will be the prefix of the media type or the weights type.
    // name of the weights or the name of the media type
    // vall-e_prompt, SD15, sd15, sdxl when implmenting add to the end
    fn get_prefix(&self)->String {
-        "valle_prompt".to_string()
+        "valle_prompt_".to_string()
    }

    // This will be ensure that the right bucket is picked
@@ -203,11 +203,11 @@ impl FileDescriptor for WeightsWorkflowDescriptor {
    }

    fn get_suffix(&self) -> String {
-        "json".to_string()
+        ".json".to_string()
    }

    fn get_prefix(&self) -> String {
-        "workflow".to_string()
+        "workflow_".to_string()
    }

    fn is_public(&self) -> bool {
@@ -58,6 +58,9 @@ pub struct ModelWeightDocument {
  pub updated_at: DateTime<Utc>,
  pub user_deleted_at: Option<DateTime<Utc>>,
  pub mod_deleted_at: Option<DateTime<Utc>>,
+
+  /// Calculated as "either user or mod deleted"
+  pub is_deleted: bool,
 }

 impl Document for ModelWeightDocument {
@@ -58,6 +58,10 @@ pub enum InferenceCategoryAbbreviated {
  #[serde(rename = "fc")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
  #[serde(alias = "format_conversion")]
  FormatConversion,
+
+  #[serde(rename = "bw")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(alias = "convert_bvh_to_workflow")]
+  ConvertBvhToWorkflow,
 }

 #[derive(Serialize, Deserialize, Clone, Copy, Debug)]
@@ -143,6 +147,7 @@ impl InferenceCategoryAbbreviated {
      InferenceCategory::Mocap => Self::Mocap,
      InferenceCategory::Workflow => Self::Workflow,
      InferenceCategory::FormatConversion => Self::FormatConversion,
+      InferenceCategory::ConvertBvhToWorkflow => Self::ConvertBvhToWorkflow,
    }
  }

@@ -156,6 +161,7 @@ impl InferenceCategoryAbbreviated {
      Self::Mocap => InferenceCategory::Mocap,
      Self::Workflow => InferenceCategory::Workflow,
      Self::FormatConversion => InferenceCategory::FormatConversion,
+      Self::ConvertBvhToWorkflow => InferenceCategory::ConvertBvhToWorkflow,
    }
  }
 }
@@ -121,6 +121,7 @@ SELECT
    jobs.on_success_result_entity_type as maybe_result_entity_type,
    jobs.on_success_result_entity_token as maybe_result_entity_token,

+    model_weights.title as maybe_model_weights_title,
    tts_models.title as maybe_tts_model_title,
    voice_conversion_models.title as maybe_voice_conversion_model_title,

@@ -144,6 +145,7 @@ SELECT

 FROM generic_inference_jobs as jobs

+LEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token
 LEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token
 LEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token

@@ -174,16 +176,25 @@ WHERE jobs.token = ?

 /// Map the internal record type (since we query over several result tables)
 fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericInferenceJobStatus {
-  let maybe_model_title = match record.inference_category {
-    InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
-    InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
-    InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
-    InferenceCategory::VideoFilter => Some("Video Filter"),
-    InferenceCategory::ImageGeneration => Some("Image Generation"),
-    InferenceCategory::Mocap => Some("Mocap"),
-    InferenceCategory::Workflow => Some("Workflow"),
-    InferenceCategory::FormatConversion => Some("format conversion"),
-  };
+  let mut maybe_model_title = None;
+
+  if let Some(title) = record.maybe_model_weights_title.as_deref() {
+    maybe_model_title = Some(title);
+  }
+
+  if maybe_model_title.is_none() {
+    maybe_model_title = match record.inference_category {
+      InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
+      InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
+      InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
+      InferenceCategory::VideoFilter => Some("Video Filter"),
+      InferenceCategory::ImageGeneration => Some("Image Generation"),
+      InferenceCategory::Mocap => Some("Mocap"),
+      InferenceCategory::Workflow => Some("Workflow"),
+      InferenceCategory::FormatConversion => Some("format conversion"),
+      InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
+    };
+  }

  // NB: A bit of a hack. We store TTS results with a full path.
  // Going forward, all other record types will store a hash.
@@ -196,6 +207,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
    InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
    InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
    InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
+    InferenceCategory::ConvertBvhToWorkflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
  };

  // NB: We've moved voice conversion out of their own table and into media_files
@@ -275,6 +287,7 @@ struct RawGenericInferenceJobStatus {
  pub maybe_result_entity_type: Option<String>,
  pub maybe_result_entity_token: Option<String>,

+  pub maybe_model_weights_title: Option<String>,
  pub maybe_tts_model_title: Option<String>,
  pub maybe_voice_conversion_model_title: Option<String>,

@@ -28,6 +28,12 @@ pub struct InsertGenericInferenceArgs<'a> {
  pub maybe_input_source_token: Option<&'a str>,
  pub maybe_input_source_token_type: Option<InferenceInputSourceTokenType>,

+  // For jobs that perform "downloads", this is the URL to download.
+  // NB: Some jobs aren't using this field yet and will pack the URL inside of
+  //   the "GenericInferenceArgs" field. The goal is to migrate them to this
+  //   top-level field eventually.
+  pub maybe_download_url: Option<&'a str>,
+
  pub maybe_raw_inference_text: Option<&'a str>,

  pub maybe_max_duration_seconds: Option<i32>,
@@ -83,6 +89,8 @@ SET
  maybe_input_source_token = ?,
  maybe_input_source_token_type = ?,

+  maybe_download_url = ?,
+
  maybe_raw_inference_text = ?,

  maybe_inference_args = ?,
@@ -115,6 +123,8 @@ SET
        args.maybe_input_source_token,
        args.maybe_input_source_token_type,

+        args.maybe_download_url,
+
        args.maybe_raw_inference_text,

        serialized_args_payload,
@@ -0,0 +1,38 @@
+use anyhow::anyhow;
+use log::error;
+use sqlx::{Executor, MySql};
+use sqlx::Error::Database;
+
+use errors::AnyhowResult;
+
+/// Insert an idempotency token (typically a UUID generated by the frontend) to make sure the
+/// frontend doesn't retry the same request by mistake. There's a unique index on idempotency
+/// tokens, so any retries with the same value will fail.
+pub async fn insert_idempotency_token<'e, 'c, E>(
+  idempotency_token: &'e str,
+  mysql_executor: E,
+) -> AnyhowResult<()>
+  where E: 'e + Executor<'c, Database=MySql>
+{
+  let query = sqlx::query!(
+    r#"INSERT INTO idempotency_tokens SET idempotency_value = ?"#,
+    idempotency_token);
+
+  let query_result = query.execute(mysql_executor).await;
+
+  match query_result {
+    Ok(_res) => Ok(()),
+    Err(err) => {
+      error!("error inserting idempotency token: {:?}", err);
+      // NB: SQLSTATE[23000]: Integrity constraint violation
+      // NB: MySQL Error Code 1062: Duplicate key insertion (this is harder to access)
+      match err {
+        Database(ref err) => {
+          let _maybe_code = err.code().map(|c| c.into_owned());
+        },
+        _ => {},
+      }
+      Err(anyhow!("error inserting idempotency token: {:?}", err))
+    }
+  }
+}
@@ -0,0 +1 @@
+pub mod insert_idempotency_token;
@@ -32,6 +32,8 @@ pub struct InsertArgs<'a> {
    pub worker_hostname: &'a str,
    pub worker_cluster: &'a str,
    pub media_file_type: MediaFileType,
+
+    pub extra_file_modification_info: Option<&'a str>,
 }

 pub async fn insert_media_file_from_comfy_ui(args: InsertArgs<'_>) -> AnyhowResult<(MediaFileToken, u64)>
@@ -92,6 +94,7 @@ SET
  creator_ip_address = ?,

  creator_set_visibility = ?,
+  extra_file_modification_info = ?,

  maybe_creator_file_synthetic_id = ?,
  maybe_creator_category_synthetic_id = ?,
@@ -122,6 +125,7 @@ SET
      args.job.creator_ip_address,

      args.job.creator_set_visibility.to_str(),
+      args.extra_file_modification_info,

      maybe_creator_file_synthetic_id,
      maybe_creator_category_synthetic_id,
@@ -84,6 +84,8 @@ SET
  maybe_origin_model_type = ?,
  maybe_origin_model_token = ?,

+  maybe_text_transcript = ?,
+
  media_type = ?,
  maybe_mime_type = ?,
  file_size_bytes = ?,
@@ -116,6 +118,8 @@ SET
      origin_model_type.to_str(),
      args.job.maybe_model_token,

+      args.job.maybe_raw_inference_text,
+
      MEDIA_TYPE.to_str(),
      args.maybe_mime_type,
      args.file_size_bytes,
@@ -81,8 +81,11 @@ pub async fn insert_media_file_from_zero_shot(
    
      origin_category = ?,
      origin_product_category = ?,
+
      maybe_origin_model_type = ?,
-    
+
+      maybe_text_transcript = ?,
+
      media_type = ?,
      maybe_mime_type = ?,
      file_size_bytes = ?,
@@ -113,6 +116,8 @@ pub async fn insert_media_file_from_zero_shot(
          ORIGIN_PRODUCT_CATEGORY.to_str(),
          args.origin_model_type.to_str(),

+          args.job.maybe_raw_inference_text,
+
          MEDIA_TYPE.to_str(),
          args.maybe_mime_type,
          args.file_size_bytes,
@@ -9,6 +9,7 @@ pub mod generic_download;
 pub mod generic_inference;
 pub mod generic_synthetic_ids;
 pub mod health_check;
+pub mod idepotency_tokens;
 pub mod ip_bans;
 pub mod media_files;
 pub mod media_uploads;
@@ -0,0 +1,296 @@
+use anyhow::anyhow;
+use chrono::{DateTime, Utc};
+use log::warn;
+use sqlx::{Executor, FromRow, MySql, QueryBuilder, Row};
+use sqlx::mysql::MySqlRow;
+
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use enums::common::visibility::Visibility;
+use enums::traits::mysql_from_row::MySqlFromRow;
+use errors::AnyhowResult;
+use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+/// This is meant to be the entire table
+#[derive(Debug)]
+pub struct ModelWeightForElasticsearchRecord {
+  pub id: i64,
+  pub token: ModelWeightToken,
+
+  pub creator_set_visibility: Visibility,
+
+  pub weights_type: WeightsType,
+  pub weights_category: WeightsCategory,
+
+  pub title: String,
+
+  // Cover images
+  pub maybe_cover_image_media_file_token: Option<MediaFileToken>,
+  pub maybe_cover_image_public_bucket_hash: Option<String>,
+  pub maybe_cover_image_public_bucket_prefix: Option<String>,
+  pub maybe_cover_image_public_bucket_extension: Option<String>,
+
+  // Creator
+  pub creator_user_token: UserToken,
+  pub creator_username: String,
+  pub creator_display_name: String,
+  pub creator_gravatar_hash: String,
+
+  // Statistics
+  pub maybe_ratings_positive_count: Option<u32>,
+  pub maybe_ratings_negative_count: Option<u32>,
+  pub maybe_bookmark_count: Option<u32>,
+
+  // TTS extensions
+  pub maybe_tts_ietf_language_tag: Option<String>,
+  pub maybe_tts_ietf_primary_language_subtag: Option<String>,
+
+  // VC extensions
+  pub maybe_voice_conversion_ietf_language_tag: Option<String>,
+  pub maybe_voice_conversion_ietf_primary_language_subtag: Option<String>,
+
+  pub created_at: DateTime<Utc>,
+  pub updated_at: DateTime<Utc>,
+
+  pub user_deleted_at: Option<DateTime<Utc>>,
+  pub mod_deleted_at: Option<DateTime<Utc>>,
+}
+
+pub async fn batch_get_model_weights_for_elastic_search_backfill<'e, 'c, E>(
+  mysql_executor: E,
+  tokens: &Vec<ModelWeightToken>,
+) -> AnyhowResult<Vec<ModelWeightForElasticsearchRecord>>
+  where E: 'e + Executor<'c, Database=MySql>
+{
+
+  let maybe_models
+      = list_model_weights(mysql_executor, tokens)
+      .await;
+
+  let models : Vec<RawRecord> = match maybe_models {
+    Ok(models) => models,
+    Err(sqlx::error::Error::RowNotFound) => return Ok(Vec::new()),
+    Err(err) => {
+      warn!("model weight list query error: {:?}", err);
+      return Err(anyhow!("model weight list query error"));
+    }
+  };
+
+  Ok(models.into_iter()
+      .map(|model| {
+        ModelWeightForElasticsearchRecord {
+          id: model.id,
+          token: model.token,
+          title: model.title,
+          weights_type: model.weights_type,
+          weights_category: model.weights_category,
+          maybe_cover_image_media_file_token: model.maybe_cover_image_media_file_token,
+          maybe_cover_image_public_bucket_hash: model.maybe_cover_image_public_bucket_hash,
+          maybe_cover_image_public_bucket_prefix: model.maybe_cover_image_public_bucket_prefix,
+          maybe_cover_image_public_bucket_extension: model.maybe_cover_image_public_bucket_extension,
+          creator_user_token: model.creator_user_token,
+          creator_username: model.creator_username,
+          creator_display_name: model.creator_display_name,
+          creator_gravatar_hash: model.creator_gravatar_hash,
+          maybe_tts_ietf_language_tag: model.maybe_tts_ietf_language_tag,
+          maybe_tts_ietf_primary_language_subtag: model.maybe_tts_ietf_primary_language_subtag,
+          maybe_voice_conversion_ietf_language_tag: model.maybe_vc_ietf_language_tag,
+          maybe_voice_conversion_ietf_primary_language_subtag: model.maybe_vc_ietf_primary_language_subtag,
+          maybe_ratings_positive_count: model.maybe_ratings_positive_count,
+          maybe_ratings_negative_count: model.maybe_ratings_negative_count,
+          maybe_bookmark_count: model.maybe_bookmark_count,
+          creator_set_visibility: model.creator_set_visibility,
+          created_at: model.created_at,
+          updated_at: model.updated_at,
+          user_deleted_at: model.user_deleted_at,
+          mod_deleted_at: model.mod_deleted_at,
+        }
+      })
+      .collect::<Vec<ModelWeightForElasticsearchRecord>>())
+}
+
+async fn list_model_weights<'e, 'c, E>(
+  mysql_executor: E,
+  tokens: &Vec<ModelWeightToken>,
+) -> Result<Vec<RawRecord>, sqlx::Error>
+  where E: 'e + Executor<'c, Database=MySql>
+{
+  if tokens.len() == 0 {
+    return Ok(Vec::new());
+  }
+
+  let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
+        r#"
+SELECT
+    w.id,
+    w.token,
+
+    w.creator_set_visibility,
+
+    w.weights_type,
+    w.weights_category,
+
+    w.title,
+
+    cover_image.token as maybe_cover_image_media_file_token,
+    cover_image.public_bucket_directory_hash as maybe_cover_image_public_bucket_hash,
+    cover_image.maybe_public_bucket_prefix as maybe_cover_image_public_bucket_prefix,
+    cover_image.maybe_public_bucket_extension as maybe_cover_image_public_bucket_extension,
+
+    w.creator_user_token,
+    users.username as creator_username,
+    users.display_name as creator_display_name,
+    users.email_gravatar_hash as creator_gravatar_hash,
+
+    entity_stats.ratings_positive_count as maybe_ratings_positive_count,
+    entity_stats.ratings_negative_count as maybe_ratings_negative_count,
+    entity_stats.bookmark_count as maybe_bookmark_count,
+
+    extension_tts.ietf_language_tag as maybe_tts_ietf_language_tag,
+    extension_tts.ietf_primary_language_subtag as maybe_tts_ietf_primary_language_subtag,
+
+    extension_vc.ietf_language_tag as maybe_vc_ietf_language_tag,
+    extension_vc.ietf_primary_language_subtag as maybe_vc_ietf_primary_language_subtag,
+
+    w.created_at,
+    w.updated_at,
+    w.user_deleted_at,
+    w.mod_deleted_at
+
+FROM model_weights as w
+
+JOIN users
+    ON users.token = w.creator_user_token
+
+LEFT OUTER JOIN model_weights_extension_tts_details as extension_tts
+    ON extension_tts.model_weights_token = w.token
+LEFT OUTER JOIN model_weights_extension_voice_conversion_details as extension_vc
+    ON extension_vc.model_weights_token = w.token
+LEFT OUTER JOIN entity_stats
+    ON entity_stats.entity_type = "model_weights"
+    AND entity_stats.entity_token = w.token
+LEFT OUTER JOIN media_files as cover_image
+    ON cover_image.token = w.maybe_cover_image_media_file_token
+
+WHERE w.token IN (
+        "#);
+
+  // NB: Syntax will be wrong if list has zero length
+  let mut separated = query_builder.separated(", ");
+
+  for token in tokens.iter() {
+    separated.push_bind(token.as_str());
+  }
+
+  separated.push_unseparated(") ");
+
+  // NB: We should return the most recent update timestamp last (for cursoring)
+  query_builder.push(" ORDER BY w.updated_at ASC, w.id ASC ");
+
+  let query = query_builder.build_query_as::<RawRecord>();
+
+  let maybe_results = query.fetch_all(mysql_executor).await?;
+
+  Ok(maybe_results)
+}
+
+struct RawRecord {
+  pub id: i64,
+  pub token: ModelWeightToken,
+
+  pub creator_set_visibility: Visibility,
+
+  pub weights_type: WeightsType,
+  pub weights_category: WeightsCategory,
+
+  pub title: String,
+
+  // Cover images
+  pub maybe_cover_image_media_file_token: Option<MediaFileToken>,
+  pub maybe_cover_image_public_bucket_hash: Option<String>,
+  pub maybe_cover_image_public_bucket_prefix: Option<String>,
+  pub maybe_cover_image_public_bucket_extension: Option<String>,
+
+  // Creator
+  pub creator_user_token: UserToken,
+  pub creator_username: String,
+  pub creator_display_name: String,
+  pub creator_gravatar_hash: String,
+
+  // Statistics
+  pub maybe_ratings_positive_count: Option<u32>,
+  pub maybe_ratings_negative_count: Option<u32>,
+  pub maybe_bookmark_count: Option<u32>,
+
+  // TTS extensions
+  pub maybe_tts_ietf_language_tag: Option<String>,
+  pub maybe_tts_ietf_primary_language_subtag: Option<String>,
+
+  // Voice conversion extensions
+  pub maybe_vc_ietf_language_tag: Option<String>,
+  pub maybe_vc_ietf_primary_language_subtag: Option<String>,
+
+  pub created_at: DateTime<Utc>,
+  pub updated_at: DateTime<Utc>,
+
+  pub user_deleted_at: Option<DateTime<Utc>>,
+  pub mod_deleted_at: Option<DateTime<Utc>>,
+}
+
+// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or
+// raw query strings) and sqlx::FromRow, regardless of whether it is derived of manually
+// implemented. Perhaps this will improve in the future, but for now manually constructed queries
+// cannot have type hints, eg. the following:
+//
+//    m.token as `token: tokens::tokens::media_files::MediaFileToken`,
+//    m.origin_category as `origin_category: enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory`,
+//    m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
+//
+// This results in the automatic mapping not being able to be found by name (for macro derive), and
+// in the manual case `row.try_get()` etc. won't have the correct column name (since the name is the
+// full "as" clause).
+impl FromRow<'_, MySqlRow> for RawRecord {
+  fn from_row(row: &MySqlRow) -> Result<Self, sqlx::Error> {
+    // NB(bt): Not sure why IDE is complaining about inlining this with the call to map() below.
+    let maybe_cover_image_media_file_token: Option<String> = row.try_get("maybe_cover_image_media_file_token")?;
+
+    Ok(Self {
+      id: row.try_get("id")?,
+      token: ModelWeightToken::new(row.try_get("token")?),
+
+      creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?,
+
+      weights_type: WeightsType::try_from_mysql_row(row, "weights_type")?,
+      weights_category: WeightsCategory::try_from_mysql_row(row, "weights_category")?,
+
+      title: row.try_get("title")?,
+
+      maybe_cover_image_media_file_token: maybe_cover_image_media_file_token.map(|token| MediaFileToken::new(token)),
+      maybe_cover_image_public_bucket_hash: row.try_get("maybe_cover_image_public_bucket_hash")?,
+      maybe_cover_image_public_bucket_prefix: row.try_get("maybe_cover_image_public_bucket_prefix")?,
+      maybe_cover_image_public_bucket_extension: row.try_get("maybe_cover_image_public_bucket_extension")?,
+
+      creator_user_token: UserToken::new(row.try_get("creator_user_token")?),
+      creator_username: row.try_get("creator_username")?,
+      creator_display_name: row.try_get("creator_display_name")?,
+      creator_gravatar_hash: row.try_get("creator_gravatar_hash")?,
+
+      maybe_ratings_positive_count: row.try_get("maybe_ratings_positive_count")?,
+      maybe_ratings_negative_count: row.try_get("maybe_ratings_negative_count")?,
+      maybe_bookmark_count: row.try_get("maybe_bookmark_count")?,
+
+      maybe_tts_ietf_language_tag: row.try_get("maybe_tts_ietf_language_tag")?,
+      maybe_tts_ietf_primary_language_subtag: row.try_get("maybe_tts_ietf_primary_language_subtag")?,
+
+      maybe_vc_ietf_language_tag: row.try_get("maybe_vc_ietf_language_tag")?,
+      maybe_vc_ietf_primary_language_subtag: row.try_get("maybe_vc_ietf_primary_language_subtag")?,
+
+      created_at: row.try_get("created_at")?,
+      updated_at: row.try_get("updated_at")?,
+      user_deleted_at: row.try_get("user_deleted_at")?,
+      mod_deleted_at: row.try_get("mod_deleted_at")?,
+    })
+  }
+}
@@ -0,0 +1 @@
+pub mod batch_get_model_weights_for_elastic_search_backfill;
@@ -33,7 +33,7 @@ pub struct RetrievedModelWeight {
    pub maybe_last_update_user_token: Option<UserToken>,
    pub original_download_url: Option<String>,
    pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
    pub file_checksum_sha2: String,
    pub public_bucket_hash: String,
    pub maybe_public_bucket_prefix: Option<String>,
@@ -271,7 +271,7 @@ pub struct RawWeight {
    pub maybe_last_update_user_token: Option<UserToken>,
    pub original_download_url: Option<String>,
    pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
    pub file_checksum_sha2: String,
    pub public_bucket_hash: String,
    pub maybe_public_bucket_prefix: Option<String>,
@@ -32,7 +32,7 @@ pub struct ModelWeightForLegacyTtsInference {
    pub maybe_last_update_user_token: Option<UserToken>,
    pub original_download_url: Option<String>,
    pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
    pub file_checksum_sha2: String,
    pub public_bucket_hash: String,
    pub maybe_public_bucket_prefix: Option<String>,
@@ -260,7 +260,7 @@ pub struct RawWeight {
    maybe_last_update_user_token: Option<UserToken>,
    original_download_url: Option<String>,
    original_filename: Option<String>,
-    file_size_bytes: i32,
+    file_size_bytes: i64,
    file_checksum_sha2: String,
    public_bucket_hash: String,
    maybe_public_bucket_prefix: Option<String>,
@@ -31,7 +31,7 @@ pub struct ModelWeightForLegacyTtsInfo {
    pub maybe_last_update_user_token: Option<UserToken>,
    pub original_download_url: Option<String>,
    pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
    pub file_checksum_sha2: String,
    pub public_bucket_hash: String,
    pub maybe_public_bucket_prefix: Option<String>,
@@ -298,7 +298,7 @@ pub struct RawWeight {
    pub maybe_last_update_user_token: Option<UserToken>,
    pub original_download_url: Option<String>,
    pub original_filename: Option<String>,
-    pub file_size_bytes: i32,
+    pub file_size_bytes: i64,
    pub file_checksum_sha2: String,
    pub public_bucket_hash: String,
    pub maybe_public_bucket_prefix: Option<String>,
@@ -0,0 +1,55 @@
+use chrono::{DateTime, Utc};
+use sqlx::{Executor, MySql, Row};
+
+use enums::by_table::model_weights::weights_types::WeightsType;
+use errors::AnyhowResult;
+use tokens::tokens::model_weights::ModelWeightToken;
+
+pub struct UpdatedModelWeight {
+  pub token: ModelWeightToken,
+  pub weights_type: WeightsType,
+  pub updated_at: DateTime<Utc>,
+}
+
+/// Get records updated since a particular date.
+/// NB: This doesn't batch yet at our scale, but in the future we'll likely need/want to batch,
+/// and then we'll need another way to cursor in the event that batch_size < # records updated
+/// within a given time quantum.
+pub async fn list_model_weight_tokens_updated_since<'e, 'c, E>(
+  mysql_executor: E,
+  since_date: &DateTime<Utc>,
+) -> AnyhowResult<Vec<UpdatedModelWeight>>
+    where E: 'e + Executor<'c, Database=MySql>
+{
+  let query = sqlx::query_as!(
+    UpdatedModelWeightRaw,
+    r#"
+      SELECT
+        token as `token: tokens::tokens::model_weights::ModelWeightToken`,
+        weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,
+        updated_at
+      FROM model_weights
+      WHERE
+      updated_at > ?
+    "#,
+    since_date
+  );
+
+  let results = query.fetch_all(mysql_executor).await?;
+
+  let results = results.into_iter()
+      .map(|record| UpdatedModelWeight {
+        token: record.token,
+        weights_type: record.weights_type,
+        updated_at: record.updated_at,
+      })
+      .collect();
+
+  Ok(results)
+}
+
+pub struct UpdatedModelWeightRaw {
+  pub token: ModelWeightToken,
+  pub weights_type: WeightsType,
+  pub updated_at: DateTime<Utc>,
+}
@@ -1,6 +1,7 @@
+pub mod list_model_weight_tokens_updated_since;
 pub mod list_model_weights_for_elastic_search_backfill_using_cursor;
 pub mod list_model_weights_for_text_to_speech;
 pub mod list_model_weights_for_voice_conversion;
 pub mod list_weights_by_tokens;
 pub mod list_weights_by_user;
-pub mod list_weights_query_builder;
+pub mod list_weights_query_builder;
@@ -1,3 +1,4 @@
+pub mod batch_get;
 pub mod create;
 pub mod delete_weights;
 pub mod edit;
@@ -55,7 +55,8 @@ fn select_result_fields() -> &'static str {
    entity_stats.bookmark_count as maybe_bookmark_count,

    media_files.media_type as maybe_media_file_type,
-    media_files.origin_product_category as maybe_media_file_origin_category,
+    media_files.origin_category as maybe_media_file_origin_category,
+    media_files.origin_product_category as maybe_media_file_origin_product,
    media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_hash,
    media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,
    media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,
@@ -3,6 +3,7 @@ use sqlx::{FromRow, Row};
 use sqlx::mysql::MySqlRow;

 use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
 use enums::by_table::media_files::media_file_type::MediaFileType;
 use enums::by_table::model_weights::weights_category::WeightsCategory;
 use enums::by_table::model_weights::weights_types::WeightsType;
@@ -93,6 +94,7 @@ pub struct RawUserBookmarkRecord {

  pub(crate) maybe_media_file_type: Option<MediaFileType>,
  pub(crate) maybe_media_file_origin_category: Option<MediaFileOriginCategory>,
+  pub(crate) maybe_media_file_origin_product: Option<MediaFileOriginProductCategory>,
  pub(crate) maybe_media_file_public_bucket_hash: Option<String>,
  pub(crate) maybe_media_file_public_bucket_prefix: Option<String>,
  pub(crate) maybe_media_file_public_bucket_extension: Option<String>,
@@ -199,6 +201,7 @@ impl FromRow<'_, MySqlRow> for RawUserBookmarkRecord {
        maybe_bookmark_count: row.try_get("maybe_bookmark_count")?,
        maybe_media_file_type: MediaFileType::try_from_mysql_row_nullable(row, "maybe_media_file_type")?,
        maybe_media_file_origin_category: MediaFileOriginCategory::try_from_mysql_row_nullable(row,"maybe_media_file_origin_category")?,
+        maybe_media_file_origin_product: MediaFileOriginProductCategory::try_from_mysql_row_nullable(row,"maybe_media_file_origin_product")?,
        maybe_media_file_public_bucket_hash: row.try_get("maybe_media_file_public_bucket_hash")?,
        maybe_media_file_public_bucket_prefix: row.try_get("maybe_media_file_public_bucket_prefix")?,
        maybe_media_file_public_bucket_extension: row.try_get("maybe_media_file_public_bucket_extension")?,
@@ -37,6 +37,10 @@ pub struct SessionUserRecord {
  pub preferred_w2l_result_visibility: Visibility,
  pub auto_play_video_preference: Option<bool>,

+  // ===== FEATURE FLAGS ===== //
+
+  pub can_access_studio: bool,
+
  // ===== ROLE ===== //

  pub user_role_slug: String,
@@ -116,6 +120,8 @@ SELECT
    users.user_role_slug,
    users.is_banned,

+    users.can_access_studio,
+
    user_roles.can_use_tts,
    user_roles.can_use_w2l,
    user_roles.can_delete_own_tts_results,
@@ -176,6 +182,8 @@ WHERE user_sessions.token = ?

        is_banned: i8_to_bool(raw_user_record.is_banned),

+        can_access_studio: i8_to_bool(raw_user_record.can_access_studio),
+
        // Usage
        can_use_tts: nullable_i8_to_bool_default_false(raw_user_record.can_use_tts),
        can_use_w2l: nullable_i8_to_bool_default_false(raw_user_record.can_use_w2l),
@@ -239,6 +247,9 @@ struct SessionUserRawDbRecord {
  user_role_slug: String,
  is_banned: i8,

+  // Feature / Rollout Flags
+  can_access_studio: i8,
+
  // NB: These are `Option` due to the JOIN not being compile-time assured.
  // Usage
  can_use_tts: Option<i8>,
@@ -0,0 +1,66 @@
+[package]
+name = "es-update-job"
+publish = false
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+
+[[bin]]
+name = "es-update-job"
+path = "src/main.rs"
+
+[dependencies]
+
+# Internal
+actix_helpers = { path = "../../../lib/actix_helpers" }
+bootstrap = { path = "../../../lib/bootstrap" }
+concurrency = { path = "../../../lib/concurrency" }
+config = { path = "../../../lib/deprecated/config" }
+container_common = { path = "../../../lib/deprecated/container_common" }
+easyenv = { path = "../../../lib/easyenv" }
+elasticsearch_schema = { path = "../../../lib/elasticsearch_schema" }
+enums = { path = "../../../_types/enums" }
+errors = { path = "../../../lib/errors" }
+filesys = { path = "../../../lib/files/filesys" }
+hashing = { path = "../../../lib/files/hashing" }
+jobs_common = { path = "../../../lib/jobs_common" }
+media = { path = "../../../lib/files/media" }
+memory_caching = { path = "../../../lib/caching/memory_caching" }
+mimetypes = { path = "../../../lib/files/mimetypes" }
+mysql_queries = { path = "../../../lib/mysql_queries" }
+redis_common = { path = "../../../lib/redis_common" }
+reusable_types = { path = "../../../_types/reusable_types" }
+server_environment = { path = "../../../lib/server_environment" }
+storyteller_root = { path = "../../../lib/storyteller_root" }
+subprocess_common = { path = "../../../lib/subprocess_common" }
+testing = { path = "../../../lib/testing" }
+tokens = { path = "../../../_types/tokens" }
+
+# Workspace hack (faster compile times)
+my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+
+# External
+anyhow = "1.0.75"
+chrono = { version = "0.4.22", features = ["serde"] }
+dotenv = "0.15.0"
+hostname = "0.3.1"
+log = "0.4.14"
+once_cell = "1.18.0"
+rand = "0.8.5"
+subprocess = "0.2.9"
+
+sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
+
+elasticsearch = "8.5.0-alpha.1"
+
+# Runtime
+# NB: We switched to Tokio from `async-std` for broader ecosystem support.
+tokio = { version = "1.34.0", features = ["macros"] }
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+serde_json = "1.0.108"
@@ -0,0 +1,2 @@
+# es-update-job default .env configurations (these can be overridden with k8s)
+
@@ -0,0 +1,4 @@
+# es-update-job default .env configurations (these can be overridden with k8s)
+
+MYSQL_URL=mysql://storyteller:password@localhost/storyteller
+ELASTICSEARCH_URL=http://localhost:9200
@@ -0,0 +1 @@
+# es-update-job default .env configurations (these can be overridden with k8s)
@@ -0,0 +1,27 @@
+use elasticsearch::Elasticsearch;
+use sqlx::MySqlPool;
+
+pub struct JobState {
+  pub mysql_pool: MySqlPool,
+  pub elasticsearch: Elasticsearch,
+
+  pub sleep_config: SleepConfigs,
+}
+
+/// Use sleep to not overload the database.
+pub struct SleepConfigs {
+  // How long to wait between individual ES writes.
+  pub between_es_writes_wait_millis: u64,
+
+  // How long to wait between individual job batches.
+  pub between_job_batch_wait_millis: u64,
+
+  // How long to wait between individual queries.
+  pub between_query_wait_millis: u64,
+
+  // How long to wait between errors.
+  pub between_error_wait_millis: u64,
+
+  // How long to wait between periods of NO-OPs / no updates.
+  pub between_no_updates_wait_millis: u64,
+}
@@ -0,0 +1,90 @@
+// Never allow these
+#![forbid(private_bounds)]
+#![forbid(private_interfaces)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+// Strict
+//#![forbid(warnings)]
+
+use elasticsearch::Elasticsearch;
+use elasticsearch::http::transport::Transport;
+use log::info;
+use sqlx::{MySql, Pool};
+use sqlx::mysql::MySqlPoolOptions;
+
+use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
+use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
+use config::shared_constants::DEFAULT_RUST_LOG;
+use errors::AnyhowResult;
+
+use crate::job_state::{JobState, SleepConfigs};
+use crate::main_loop::main_loop;
+
+pub mod job_state;
+pub mod main_loop;
+
+#[tokio::main]
+async fn main() -> AnyhowResult<()> {
+
+  let container_environment = bootstrap(BootstrapArgs {
+    app_name: "es-update-job",
+    default_logging_override: Some(DEFAULT_RUST_LOG),
+    config_search_directories: &[".", "./config", "crates/service/job/es-update-job/config"],
+  })?;
+
+  info!("Hostname: {}", &container_environment.hostname);
+
+  let mysql_pool = get_mysql_pool().await?;
+  let elasticsearch = get_elasticsearch_client()?;
+
+
+  let job_state = JobState {
+    mysql_pool,
+    elasticsearch,
+    sleep_config: SleepConfigs {
+      between_es_writes_wait_millis: easyenv::get_env_num("BETWEEN_WRITES_WAIT_MILLIS", 100)?,
+      between_job_batch_wait_millis: easyenv::get_env_num("BETWEEN_JOB_BATCH_WAIT_MILLIS", 500)?,
+      between_query_wait_millis: easyenv::get_env_num("BETWEEN_QUERY_WAIT_MILLIS", 100)?,
+      between_error_wait_millis: easyenv::get_env_num("BETWEEN_ERROR_WAIT_MILLIS", 10_000)?,
+      between_no_updates_wait_millis: easyenv::get_env_num("BETWEEN_NO_UPDATES_WAIT_MILLIS", 20_000)?,
+    },
+  };
+
+  let _r = main_loop(job_state).await;
+
+  Ok(())
+}
+
+async fn get_mysql_pool() -> AnyhowResult<Pool<MySql>> {
+  info!("Connecting to MySQL database...");
+
+  let db_connection_string =
+      easyenv::get_env_string_or_default(
+        "MYSQL_URL",
+        DEFAULT_MYSQL_CONNECTION_STRING);
+
+  let mysql_pool = MySqlPoolOptions::new()
+      .max_connections(5)
+      .connect(&db_connection_string)
+      .await?;
+
+  Ok(mysql_pool)
+}
+
+fn get_elasticsearch_client() -> AnyhowResult<Elasticsearch> {
+  info!("Connecting to Elasticsearch...");
+  let transport = Transport::single_node(&easyenv::get_env_string_required("ELASTICSEARCH_URL")?)?;
+
+  // TODO(bt,2023-10-26): Allow connecting to instances by URL instead of the default dev URL.
+  Ok(Elasticsearch::new(transport))
+}
@@ -0,0 +1,183 @@
+use std::time::Duration;
+
+use anyhow::anyhow;
+use chrono::{DateTime, Utc};
+use elasticsearch::{BulkOperation, BulkParts, Elasticsearch};
+use log::{error, info};
+use serde_json::Value;
+
+use elasticsearch_schema::documents::model_weight_document::{MODEL_WEIGHT_INDEX, ModelWeightDocument};
+use elasticsearch_schema::traits::document::Document;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use errors::AnyhowResult;
+use mysql_queries::queries::model_weights::batch_get::batch_get_model_weights_for_elastic_search_backfill::{batch_get_model_weights_for_elastic_search_backfill, ModelWeightForElasticsearchRecord};
+use mysql_queries::queries::model_weights::list::list_model_weight_tokens_updated_since::list_model_weight_tokens_updated_since;
+
+use crate::job_state::JobState;
+
+pub async fn main_loop(job_state: JobState) {
+  // TODO(bt,2024-02-05): Write this cursor to Redis so job can resume without reindexing everything.
+  let mut cursor = DateTime::UNIX_EPOCH;
+
+  loop {
+    info!("Main loop; cursor @ {:?}", &cursor);
+
+    let result = with_database_main_loop(&mut cursor, &job_state).await;
+
+    if let Err(err) = result {
+      error!("Error in main loop: {:?}", err);
+      std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_error_wait_millis));
+    }
+
+    // NB: Function should never return in success case, but we'll sleep just in case.
+  }
+}
+
+pub async fn with_database_main_loop(updated_at_cursor: &mut DateTime<Utc>, job_state: &JobState) -> AnyhowResult<()> {
+  info!("Acquiring MySQL connection...");
+
+  let mut mysql_connection = job_state.mysql_pool.acquire().await?;
+
+  let mut last_successful_update_at = updated_at_cursor.clone();
+
+  loop {
+    info!("Querying tokens updated since: {:?}", &updated_at_cursor);
+
+    let mut maybe_tokens =
+        list_model_weight_tokens_updated_since(&mut *mysql_connection, &updated_at_cursor).await?;
+
+    if maybe_tokens.is_empty() {
+      info!("No records updated since {:?}", &updated_at_cursor);
+      std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_no_updates_wait_millis));
+      continue;
+    }
+
+    info!("Found {} updated records", maybe_tokens.len());
+
+    let mut last_observed_updated_at = updated_at_cursor.clone();
+
+    while !maybe_tokens.is_empty() {
+      // NB: This list might be very large if we query from (1) the epoch, or (2) there was a large series of updates
+      let last = 50.min(maybe_tokens.len());
+      let drained_tokens = maybe_tokens.drain(0..last)
+          .into_iter()
+          .map(|record| record.token)
+          .collect::<Vec<_>>();
+
+      let records
+          = batch_get_model_weights_for_elastic_search_backfill(&mut *mysql_connection, &drained_tokens).await?;
+
+      for record in records {
+        let updated_at = record.updated_at.clone();
+
+        create_document_from_record(&job_state.elasticsearch, record).await?;
+
+        // NB: We don't want to advance the cursor to the current second just yet, because we might be processing
+        // several records with the exact same "updated_at" timestamp. We do know that due to ordering, we can update
+        // the timestamp cursor to the second before this timestamp, however.
+        //
+        // If we change the cursoring or batch sizes, we'll need to reconsider all of this logic.
+        if let Some(cursor_at_least) = updated_at.checked_sub_signed(chrono::Duration::seconds(1)) {
+          // NB: We don't want the cursor to slide backwards (it shouldn't, but the max will save us).
+          last_successful_update_at = cursor_at_least.max(last_successful_update_at);
+        }
+
+        *updated_at_cursor = last_successful_update_at.max(*updated_at_cursor);
+
+        last_observed_updated_at = updated_at;
+
+        std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_es_writes_wait_millis));
+      }
+
+      std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_job_batch_wait_millis));
+    }
+
+    // NB: The last cursor math put us a second behind the current clock. Here we'll advance to the last record.
+    // NB: This technically could miss records if we're updating within the same second batch we read from, but that
+    // seems both unlikely and not worth solving at our present scale.
+    *updated_at_cursor = last_observed_updated_at.max(*updated_at_cursor);
+
+    info!("Up to date at cursor = {:?}", &updated_at_cursor);
+
+    std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_query_wait_millis));
+  }
+}
+
+async fn create_document_from_record(elasticsearch: &Elasticsearch, record: ModelWeightForElasticsearchRecord) -> AnyhowResult<()> {
+  info!("Create record for {:?} - {:?}", record.token, record.title);
+
+  let is_deleted = record.user_deleted_at.is_some() || record.mod_deleted_at.is_some();
+
+  let document = ModelWeightDocument {
+    token: record.token,
+
+    creator_set_visibility: record.creator_set_visibility,
+
+    weights_type: record.weights_type,
+    weights_category: record.weights_category,
+
+    title: record.title,
+
+    maybe_cover_image_media_file_token: record.maybe_cover_image_media_file_token,
+    maybe_cover_image_public_bucket_hash: record.maybe_cover_image_public_bucket_hash,
+    maybe_cover_image_public_bucket_prefix: record.maybe_cover_image_public_bucket_prefix,
+    maybe_cover_image_public_bucket_extension: record.maybe_cover_image_public_bucket_extension,
+
+    creator_user_token: record.creator_user_token,
+    creator_username: record.creator_username,
+    creator_display_name: record.creator_display_name,
+    creator_gravatar_hash: record.creator_gravatar_hash,
+
+    ratings_positive_count: record.maybe_ratings_positive_count.unwrap_or(0),
+    ratings_negative_count: record.maybe_ratings_negative_count.unwrap_or(0),
+    bookmark_count: record.maybe_bookmark_count.unwrap_or(0),
+
+    maybe_ietf_language_tag: match record.weights_category {
+      WeightsCategory::TextToSpeech => record.maybe_tts_ietf_language_tag,
+      WeightsCategory::VoiceConversion => record.maybe_voice_conversion_ietf_language_tag,
+      _ => None,
+    },
+
+    maybe_ietf_primary_language_subtag: match record.weights_category {
+      WeightsCategory::TextToSpeech => record.maybe_tts_ietf_primary_language_subtag,
+      WeightsCategory::VoiceConversion => record.maybe_voice_conversion_ietf_primary_language_subtag,
+      _ => None,
+    },
+
+    created_at: record.created_at,
+    updated_at: record.updated_at,
+    user_deleted_at: record.user_deleted_at,
+    mod_deleted_at: record.mod_deleted_at,
+
+    is_deleted,
+  };
+
+  let op : BulkOperation<_> = BulkOperation::index(&document)
+      .id(document.get_document_id())
+      .into();
+
+  let response = elasticsearch
+      .bulk(BulkParts::Index(MODEL_WEIGHT_INDEX))
+      .body(vec![op])
+      .send()
+      .await?;
+
+  let json: Value = response.json().await?;
+
+  let had_errors = json["errors"].as_bool().unwrap_or(false);
+
+  if had_errors {
+    let failed: Vec<&Value> = json["items"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|v| !v["error"].is_null())
+        .collect();
+
+    error!("Errors during indexing. Failures: {}", failed.len());
+
+    return Err(anyhow!("Errors during indexing. Failures: {}", failed.len()));
+  }
+
+  Ok(())
+}
@@ -84,4 +84,7 @@ mime = "0.3.17"
 rand = { version = "0.8.5", features = [] }
 jsonpath_lib = "0.3.0"
 walkdir = "2.4.0"
-reqwest = "0.11.22"
+reqwest = "0.11.22"
+opentelemetry = {version = "0.21.0", features = ["metrics"]}
+opentelemetry-otlp = { version = "0.14.0", features = ["metrics"] }
+opentelemetry_sdk = { version = "0.21.2", features = ["metrics", "rt-tokio"] }
@@ -9,6 +9,9 @@ HTTP_BIND_ADDRESS=0.0.0.0:11223
 # NB: Disable if not testing Redis
 REDIS_FOR_KEEPALIVE_URL="redis://localhost/0"

+# General model weights cache directory
+MODEL_WEIGHTS_CACHE_DIR="/tmp/model_weights_cache"
+
 # ================================= #
 #                                   #
 #            Vall-E-X               #
@@ -28,9 +31,16 @@ VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .venv/bin/activate"
 #VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .env/bin/activate"
 #VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE="" # fill this out for dev?

-STYLE_TTS2_INFERENCE_ROOT_DIRECTORY=""
-STYLE_TTS2_INFERENCE_COMMAND = ""
-STYLE_TTS2_INFERENCE_EXECUTABLE = ""
+# ================================= #
+#                                   #
+#           STYLE TTS 2             #
+#                                   #
+# ================================= #
+
+# TODO(bt,2024-02-10): Not sure what these should be
+STYLE_TTS2_INFERENCE_EXECUTABLE=""
+STYLE_TTS2_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/tts/style-tts2"
+STYLE_TTS2_INFERENCE_COMMAND="venv/bin/python3.8 main.py"

 # ================================= #
 #                                   #
@@ -104,6 +114,7 @@ VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b"
 #            Rerender               #
 #                                   #
 # ================================= #
+
 RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/"
 RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py"

@@ -112,8 +123,15 @@ RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py"
 #          STABLE DIFFUSION         #
 #                                   #
 # ================================= #
+
 STABLE_DIFFUSION_INFERENCE_ROOT_DIRECTORY="/home/tensor/code/storyteller/storyteller-ml/image_generation/Storyteller-Studio-Filter/"
 STABLE_DIFFUSION_INFERENCE_COMMAND=".venv/bin/python main.py"
+SD_VAE_BUCKET_PATH="/stable_diffusion_deps/vaeFtEma560000Ema_original.pt"
+SD_PREDEFINED_SD_WEIGHT_BUCKET_PATH="/stable_diffusion_deps/fiamixRebootPastel_v14LightColor.safetensors"
+
+# TODO(bt,2024-02-10): Remove these
+SD_PREDEFINED_SD_WEIGHT_TOKEN="INVALID_VALUE_REMOVE"
+SD_PREDEFINED_LORA_WEIGHT_TOKEN="INVALID_VALUE_REMOVE"

 # ================================= #
 #                                   #
@@ -130,18 +148,30 @@ MOCAPNET_INFERENCE_EXECUTABLE_OR_COMMAND="pythonVirtualEnvironment/bin/python -m
 # ================================= #
 COMFY_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/image_generation/ComfyUI"
 COMFY_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python ../ComfyLauncher/ComfyRunner.py"
-STYLE_TTS2_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/tts/style-tts2"
-STYLE_TTS2_INFERENCE_COMMAND="venv/bin/python3.8 main.py"
+
+# TODO(bt,2024-02-10): Not sure what these should be
+COMFY_MOUNTS_DIRECTORY="NOT_SURE_YET"
+COMFY_WORKFLOW_BUCKET_PATH="NOT_SURE_YET"

 # ================================= #
 #                                   #
 #            Fbx2Gltf               #
 #                                   #
 # ================================= #
+
 #FBX2GLTF_DIRECTORY="%storyteller_root%/FBX2glTF"
 FBX2GLTF_DIRECTORY="/home/bt/dev/storyteller/FBX2glTF"
 FBX2GLTF_EXECUTABLE_OR_COMMAND="./bin/FBX2glTF-linux-x64"

+# ================================= #
+#                                   #
+#         BEVY COMPOSITOR           #
+#                                   #
+# ================================= #
+
+# TODO(bt,2024-02-10): Not sure what these should be
+BVH2WORKFLOW_EXECUTABLE_OR_COMMAND="NOT_SURE_YET"
+
 # ================================= #
 #                                   #
 #          CACHE SETTINGS           #
@@ -9,13 +9,17 @@ use jobs_common::noop_logger::NoOpLogger;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::{AvailableInferenceJob, list_available_generic_inference_jobs, ListAvailableGenericInferenceJobArgs};
 use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed;
 use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_failure::mark_generic_inference_job_failure;
+use opentelemetry::metrics::AsyncInstrument;

+use crate::OTEL_METER_NAME;
 use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem;
 use crate::job::job_loop::process_single_job::process_single_job;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
 use crate::job_dependencies::JobDependencies;

+use opentelemetry::{global as otel, KeyValue as OtelAttribute, metrics::Unit};
+
 // Job runner timeouts (guards MySQL)
 const START_TIMEOUT_MILLIS : u64 = 500;
 const INCREASE_TIMEOUT_MILLIS : u64 = 1000;
@@ -61,6 +65,7 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
    }).await;

    let batch_query_duration = Instant::now().duration_since(batch_query_start_time);
+    job_dependencies.job_instruments.batch_query_duration.record(batch_query_duration.as_millis() as u64, &[]);

    if batch_query_duration > SLOW_BATCH_QUERY_NOTICE_DURATION {
      warn!("Batch query took duration: {:?}", &batch_query_duration);
@@ -91,10 +96,16 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
      continue;
    }

+    job_dependencies.job_instruments.batch_size.record(jobs.len() as u64, &[]);
    info!("Queried {} jobs from database", jobs.len());

+    let batch_processing_start_time = Instant::now();
+
    let batch_result = process_job_batch(&job_dependencies, jobs).await;

+    let batch_processing_duration = Instant::now().duration_since(batch_processing_start_time);
+    job_dependencies.job_instruments.batch_processing_duration.record(batch_processing_duration.as_millis() as u64, &[]);
+
    match batch_result {
      Ok(_) => {},
      Err(e) => {
@@ -118,9 +129,14 @@ pub async fn main_loop(job_dependencies: JobDependencies) {

 async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<AvailableInferenceJob>) -> AnyhowResult<()> {
  let job_count = jobs.len();
+
  for (i, job) in jobs.into_iter().enumerate() {
+    job_dependencies.job_instruments.total_job_count.add(1, &[]);
+
+    let start_time = Instant::now();
    let result = process_single_job(job_dependencies, &job).await;

+
    match result {
      Ok(success_case) => {
        info!("Job loop iteration ({i} of {job_count} batch) \"success\": {:?}", success_case);
@@ -134,6 +150,7 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab

        if increment_success_count {
          let stats = job_dependencies.job.info.job_stats.increment_success_count().ok();
+          job_dependencies.job_instruments.job_success_count.add(1, &[]);
          warn!("Success stats: {:?}", stats);
        }
      },
@@ -145,6 +162,21 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
        let _r = handle_error(&job_dependencies, &job, err).await?;
      }
    }
+    let job_duration = Instant::now().duration_since(start_time);;
+
+    let model_type_str = match job.maybe_model_type {
+      Some(model_type) => model_type.to_string(),
+      None => "unknown".to_string(),
+    };
+
+    job_dependencies.job_instruments.job_duration.record(job_duration.as_millis() as u64,
+      &[
+        OtelAttribute::new("job_user_is_premium", job.is_from_premium_user),
+        OtelAttribute::new("job_model", model_type_str),
+        OtelAttribute::new("job_status", job.status.to_str()),
+        OtelAttribute::new("job_inference_category", job.inference_category.to_str()),
+      ]
+    );
  }

  Ok(())
@@ -256,6 +288,10 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere
    warn!("Failure stats: {:?}", stats);
  }

+  job_dependencies.job_instruments.job_failure_count.add(1, &[
+    OtelAttribute::new("job_failure_internal_reason", internal_failure_reason.clone()),
+  ]);
+
  match job_failure_class {
    JobFailureClass::PermanentFailure => {
      let _r = mark_generic_inference_job_completely_failed(
@@ -13,6 +13,7 @@ use redis_common::redis_keys::RedisKeys;
 use crate::job::job_loop::determine_dependency_status::determine_dependency_status;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
+use crate::job::job_types::bevy_to_workflow::process_single_bevy_to_workflow_conversion_job::process_single_bevy_to_workflow_conversion_job;
 use crate::job::job_types::format_conversion::process_single_format_conversion_job::process_single_format_conversion_job;
 use crate::job::job_types::image_generation::process_single_ig_job::process_single_ig_job;
 use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
@@ -202,6 +203,9 @@ async fn do_process_single_job(
    InferenceCategory::FormatConversion => {
      process_single_format_conversion_job(job_dependencies, job).await?
    }
+    InferenceCategory::ConvertBvhToWorkflow => {
+      process_single_bevy_to_workflow_conversion_job(job_dependencies, job).await?
+    }
  };

  let maybe_entity_type = job_success_result.maybe_result_entity
@@ -234,6 +238,8 @@ async fn do_process_single_job(

  info!("Saved model record: {} - {}", job.id.0, &job.inference_job_token);

+  job_dependencies.job_instruments.inference_command_execution_duration.record(inference_duration.as_millis() as u64, &[]);
+
  // TODO(bt, 2023-01-11): Need to publish that the job finished.
  //  Publish the *correct type* of event.
  //job_dependencies.firehose_publisher.publish_generic_download_finished(
@@ -0,0 +1,23 @@
+use std::path::Path;
+
+use filesys::path_to_string::path_to_string;
+use subprocess_common::command_runner::command_args::CommandArgs;
+
+pub struct BvhToWorkflowCommandArgs<'a> {
+    pub input_file: &'a Path,
+    pub output_directory: &'a Path,
+}
+
+impl CommandArgs for BvhToWorkflowCommandArgs<'_> {
+    fn to_command_string(&self) -> String {
+        let mut command = String::new();
+
+        command.push_str(" -i ");
+        command.push_str(&path_to_string(self.input_file));
+
+        command.push_str(" -o ");
+        command.push_str(&path_to_string(self.output_directory));
+
+        command
+    }
+}
@@ -0,0 +1,46 @@
+use std::path::PathBuf;
+
+use errors::AnyhowResult;
+use subprocess_common::command_runner::command_runner::CommandRunner;
+use subprocess_common::command_runner::env_var_policy::EnvVarPolicy;
+use subprocess_common::docker_options::{DockerFilesystemMount, DockerOptions};
+use subprocess_common::executable_or_command::ExecutableOrShellCommand;
+
+pub struct BvhToWorkflowDependencies {
+    pub command_runner: CommandRunner,
+}
+
+impl BvhToWorkflowDependencies {
+    pub fn setup() -> AnyhowResult<Self> {
+        let maybe_execution_directory = easyenv::get_env_pathbuf_optional("BVH2WORKFLOW_DIRECTORY");
+
+        let executable_or_command = easyenv::get_env_string_required("BVH2WORKFLOW_EXECUTABLE_OR_COMMAND")?;
+
+        let executable_or_command = if executable_or_command.contains(" ") {
+            ExecutableOrShellCommand::BashShellCommand(executable_or_command)
+        } else {
+            ExecutableOrShellCommand::Executable(PathBuf::from(executable_or_command))
+        };
+
+        let maybe_execution_timeout = easyenv::get_env_duration_seconds_optional("BVH2WORKFLOW_TIMEOUT_SECONDS");
+
+        let maybe_docker_options = easyenv::get_env_string_optional("BVH2WORKFLOW_DOCKER_IMAGE")
+            .map(|image_name| DockerOptions {
+                image_name,
+                maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+                maybe_environment_variables: None,
+                maybe_gpu: None,
+            });
+
+        Ok(Self {
+            command_runner: CommandRunner {
+                executable_or_command,
+                maybe_execution_directory,
+                env_var_policy: EnvVarPolicy::CopyNone,
+                maybe_virtual_env_activation_command: None,
+                maybe_docker_options,
+                maybe_execution_timeout,
+            }
+        })
+    }
+}
@@ -0,0 +1,3 @@
+pub mod process_job;
+pub mod command_args;
+pub mod dependencies;
@@ -0,0 +1,290 @@
+use std::fs::read_to_string;
+use std::time::Instant;
+
+use anyhow::anyhow;
+use log::{error, info, warn};
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
+
+use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use filesys::check_file_exists::check_file_exists;
+use filesys::file_size::file_size;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
+use hashing::sha256::sha256_hash_file::sha256_hash_file;
+use mimetypes::mimetype_for_file::get_mimetype_for_file;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::{insert_media_file_generic, InsertArgs};
+use mysql_queries::queries::media_files::get_media_file_for_inference::MediaFileForInference;
+use subprocess_common::command_runner::command_runner_args::{FileOrCreate, RunAsSubprocessArgs};
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::command_args::BvhToWorkflowCommandArgs;
+use crate::job_dependencies::JobDependencies;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
+
+const BUCKET_FILE_EXTENSION : &str = ".mp4";
+const BUCKET_FRAMES_FILE_EXTENSION : &str = ".zip";
+
+pub struct BvhToWorkflowJobArgs<'a> {
+  pub job_dependencies: &'a JobDependencies,
+  pub job: &'a AvailableInferenceJob,
+  pub media_file: &'a MediaFileForInference,
+}
+
+pub async fn process_job(args: BvhToWorkflowJobArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
+  let job = args.job;
+  let media_file = args.media_file;
+
+  let mut job_progress_reporter = args.job_dependencies
+      .clients
+      .job_progress_reporter
+      .new_generic_inference(job.inference_job_token.as_str())
+      .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
+
+  let model_dependencies = args
+      .job_dependencies
+      .job
+      .job_specific_dependencies
+      .maybe_convert_bvh_to_workflow_dependencies
+      .as_ref()
+      .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("missing BvhToWorkflow dependencies".to_string())))?;
+
+  // ==================== TEMP DIR ==================== //
+
+  let work_temp_dir = format!("temp_file_convert_{}", job.id.0);
+
+  // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
+  let work_temp_dir = args.job_dependencies
+      .fs
+      .scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  // ==================== DOWNLOAD MEDIA FILE ==================== //
+
+  info!("Download media for conversion...");
+
+  let original_media_upload_fs_path = {
+    let original_media_file_fs_path = work_temp_dir.path().join("original.bvh");
+
+    let media_file_bucket_path = MediaFileBucketPath::from_object_hash(
+      &media_file.public_bucket_directory_hash,
+      media_file.maybe_public_bucket_prefix.as_deref(),
+      media_file.maybe_public_bucket_extension.as_deref());
+
+    let bucket_object_path = media_file_bucket_path.to_full_object_pathbuf();
+
+    info!("Downloading media to bucket path: {:?}", &bucket_object_path);
+
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "media upload (original file)",
+      final_filesystem_file_path: &original_media_file_fs_path,
+      bucket_object_path: &bucket_object_path,
+      bucket_client: &args.job_dependencies.buckets.public_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;
+
+    original_media_file_fs_path
+  };
+
+  // ==================== SETUP FOR CONVERSION ==================== //
+
+  job_progress_reporter.log_status("running conversion")
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+
+  // ==================== RUN INFERENCE SCRIPT ==================== //
+
+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+  let output_directory = work_temp_dir.path().join("output");
+  let output_directory_actual = work_temp_dir.path();
+
+  let execution_start_time = Instant::now();
+
+  let command_exit_status = {
+    model_dependencies
+        .command_runner
+        .run_with_subprocess(RunAsSubprocessArgs {
+          args: Box::new(&BvhToWorkflowCommandArgs {
+            input_file: &original_media_upload_fs_path,
+            output_directory: &output_directory,
+          }),
+          maybe_stderr_output_file: Some(FileOrCreate::NewFileWithName(&stderr_output_file)),
+          maybe_stdout_output_file: None,
+        })
+  };
+
+  let execution_duration = Instant::now().duration_since(execution_start_time);
+
+  info!("Execution took duration to complete: {:?}", &execution_duration);
+
+  if !command_exit_status.is_success() {
+    error!("Execution failed: {:?}", command_exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+    }
+
+    safe_delete_temp_file(&original_media_upload_fs_path);
+    safe_delete_temp_directory(&output_directory_actual);
+    safe_delete_temp_directory(&work_temp_dir);
+
+    return Err(error);
+  }
+
+  // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
+
+  let output_video_file = output_directory_actual.join("output.mp4");
+  let output_frames_zip_file = output_directory_actual.join("output.zip");
+  // NB: The actual name of the file will be this:
+
+  info!("Checking that output files exist...");
+
+  check_file_exists(&output_video_file).map_err(|e| ProcessSingleJobError::Other(e))?;
+  check_file_exists(&output_frames_zip_file).map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!("Interrogating result file properties...");
+
+  let file_size_bytes = file_size(&output_video_file)
+      .map_err(|err| ProcessSingleJobError::Other(err))?;
+
+  let maybe_mimetype = get_mimetype_for_file(&output_video_file)
+      .map_err(|err| ProcessSingleJobError::from_io_error(err))?
+      .map(|mime| mime.to_string());
+
+  info!("Calculating sha256...");
+
+  let file_checksum = sha256_hash_file(&output_video_file)
+      .map_err(|err| {
+        ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err))
+      })?;
+
+  // ==================== UPLOAD AUDIO TO BUCKET ==================== //
+
+  job_progress_reporter.log_status("uploading result")
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  let result_video_bucket_location = MediaFileBucketPath::generate_new(
+    None,
+    Some(BUCKET_FILE_EXTENSION));
+
+  let result_bucket_object_pathbuf = result_video_bucket_location.to_full_object_pathbuf();
+
+  info!("Destination bucket path: {:?}", &result_bucket_object_pathbuf);
+
+  info!("Uploading media file...");
+
+  args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type(
+    &result_bucket_object_pathbuf,
+    &output_video_file,
+    "video/mp4")
+      .await
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  let result_zip_bucket_location = MediaFileBucketPath::from_object_hash(&result_video_bucket_location.get_object_hash(),
+    None,
+    Some(".zip"));
+
+  let result_bucket_object_pathbuf = result_zip_bucket_location.to_full_object_pathbuf();
+
+  info!("Destination bucket path: {:?}", &result_bucket_object_pathbuf);
+
+  info!("Uploading media file...");
+
+  args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type(
+    &result_bucket_object_pathbuf,
+    &output_frames_zip_file,
+  "application/zip")
+      .await
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  safe_delete_temp_file(&output_video_file);
+  safe_delete_temp_file(&output_frames_zip_file);
+  safe_delete_temp_directory(&output_directory_actual);
+
+  // ==================== DELETE DOWNLOADED FILE ==================== //
+
+  // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk...
+  safe_delete_temp_directory(&work_temp_dir);
+
+  // ==================== SAVE RECORDS ==================== //
+
+  info!("Saving record...");
+
+  let (inference_result_token, id) = insert_media_file_generic(InsertArgs {
+    pool: &args.job_dependencies.db.mysql_pool,
+    job: &job,
+    media_type: MediaFileType::Video,
+    origin_category: MediaFileOriginCategory::Processed,
+    origin_product_category: MediaFileOriginProductCategory::Mocap,
+    maybe_origin_model_type: None,
+    maybe_origin_model_token: None,
+    maybe_origin_filename: None,
+    maybe_mime_type: maybe_mimetype.as_deref(),
+    file_size_bytes,
+    maybe_duration_millis: None,
+    maybe_audio_encoding: None,
+    maybe_video_encoding: None,
+    maybe_frame_width: None,
+    maybe_frame_height: None,
+    public_bucket_directory_hash: result_video_bucket_location.get_object_hash(),
+    maybe_public_bucket_prefix: result_video_bucket_location.get_optional_prefix(),
+    maybe_public_bucket_extension: result_video_bucket_location.get_optional_extension(),
+    extra_file_modification_info: None,
+    maybe_creator_file_synthetic_id_category: IdCategory::MediaFile,
+    maybe_creator_category_synthetic_id_category: IdCategory::MocapResult,
+    maybe_mod_user_token: None,
+    is_batch_generated: false,
+    is_generated_on_prem: false,
+    checksum_sha2: &file_checksum,
+    generated_by_worker: Some(&args.job_dependencies.job.info.container.hostname),
+    generated_by_cluster: Some(&args.job_dependencies.job.info.container.cluster_name),
+  })
+      .await
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!("VC Done.");
+
+  //args.job_dependencies.clients.firehose_publisher.vc_inference_finished(
+  //  maybe_user_token.as_ref(),
+  //  &job.inference_job_token,
+  //  inference_result_token.as_str())
+  //    .await
+  //    .map_err(|e| {
+  //      error!("error publishing event: {:?}", e);
+  //      ProcessSingleJobError::Other(anyhow!("error publishing event"))
+  //    })?;
+
+  job_progress_reporter.log_status("done")
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  info!("Job {:?} complete success! Downloaded, executed successfully, and uploaded. Saved record: {}, Result Token: {}",
+        job.id, id, &inference_result_token);
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::MediaFile,
+      entity_token: inference_result_token.to_string(),
+    }),
+    inference_duration: execution_duration,
+  })
+}
+
+#[derive(Deserialize, Default)]
+struct FileMetadata {
+  pub duration_millis: Option<u64>,
+  pub mimetype: Option<String>,
+  pub file_size_bytes: u64,
+}
@@ -0,0 +1,2 @@
+pub mod bvh_to_workflow;
+pub mod process_single_bevy_to_workflow_conversion_job;
@@ -0,0 +1,50 @@
+use anyhow::anyhow;
+use log::error;
+
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::get_media_file_for_inference::get_media_file_for_inference;
+use tokens::tokens::media_files::MediaFileToken;
+
+use crate::job::job_loop::job_success_result::JobSuccessResult;
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow;
+use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::process_job::BvhToWorkflowJobArgs;
+use crate::job_dependencies::JobDependencies;
+
+
+pub async fn process_single_bevy_to_workflow_conversion_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let maybe_media_file_token = job.maybe_input_source_token
+      .as_deref()
+      .map(|token| MediaFileToken::new_from_str(token));
+
+  let media_file_token = match maybe_media_file_token {
+    None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media file for format conversion job: {:?}", job.inference_job_token))),
+    Some(token) => token,
+  };
+
+  let maybe_media_file_result =
+      get_media_file_for_inference(&media_file_token, &job_dependencies.db.mysql_pool).await;
+
+  let media_file = match maybe_media_file_result {
+    Ok(Some(media_file)) => media_file,
+    Ok(None) => {
+      error!("no media file record found for token: {:?}", media_file_token);
+      return Err(ProcessSingleJobError::Other(anyhow!("no media file record found for token: {:?}", media_file_token)));
+    }
+    Err(err) => {
+      error!("error fetching media file record from db: {:?}", err);
+      return Err(ProcessSingleJobError::Other(err));
+    }
+  };
+
+  let job_success_result = bvh_to_workflow::process_job::process_job(BvhToWorkflowJobArgs {
+        job_dependencies,
+        job,
+        media_file: &media_file,
+      }).await?;
+
+  Ok(job_success_result)
+}
+
+
@@ -3,4 +3,7 @@ pub mod process_job;
 pub mod model_downloaders;
 pub mod sd_inference_command;
 pub mod stable_diffusion_dependencies;
-pub mod validate_inputs;
+pub mod validate_inputs;
+mod process_job_inference;
+mod process_job_lora_upload;
+mod process_job_sd_upload;
@@ -1,49 +1,16 @@
-use std::fs::read_to_string;
-use std::path::PathBuf;
-use std::time::{Duration, Instant};
-
 use actix_web::dev::ResourcePath;
 use anyhow::anyhow;
-use log::{error, info, warn};
 use serde_json;

-use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
-use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
-use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
-use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
-use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
-use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
-use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
-use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
-use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
-use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
-use enums::by_table::media_files::media_file_type::MediaFileType;
-use enums::by_table::model_weights::weights_category::WeightsCategory;
-use enums::by_table::model_weights::weights_types::WeightsType;
-use filesys::file_exists::file_exists;
-use filesys::path_to_string::path_to_string;
-use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
-use filesys::safe_delete_temp_file::safe_delete_temp_file;
-use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
-use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
 use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
-use mysql_queries::queries::batch_generations::insert_batch_generation_records::insert_batch_generation_records;
-use mysql_queries::queries::batch_generations::insert_batch_generation_records::InsertBatchArgs;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::media_files::create::insert_media_file_generic::insert_media_file_generic;
-use mysql_queries::queries::media_files::create::insert_media_file_generic::InsertArgs;
-use mysql_queries::queries::model_weights::create::create_weight::{
-    create_weight,
-    CreateModelWeightsArgs,
-};
-use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
-use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
-use tokens::tokens::model_weights::ModelWeightToken;
-use tokens::tokens::users::UserToken;

-use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
-use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job::job_types::image_generation::sd::process_job_inference::process_job_inference;
+use crate::job::job_types::image_generation::sd::process_job_lora_upload::process_job_lora_upload;
+use crate::job::job_types::image_generation::sd::process_job_sd_upload::process_job_sd_upload;
 use crate::job_dependencies::JobDependencies;

 #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
@@ -72,11 +39,7 @@ pub async fn sd_args_from_job(
 ) -> Result<StableDiffusionArgs, ProcessSingleJobError> {
    let inference_args = args.job.maybe_inference_args
        .as_ref()
-        .map(
-            |
-                args: &mysql_queries::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs
-            | args.args.as_ref()
-        )
+        .map(|args: &GenericInferenceArgs| args.args.as_ref())
        .flatten();

    let polymorphic_args = match inference_args {
@@ -110,868 +73,14 @@ pub async fn process_job_selection(
    if sd_args.type_of_inference == "inference" {
        process_job_inference(&args).await
    } else if sd_args.type_of_inference == "lora" {
-        process_job_lora(&args).await
+        process_job_lora_upload(&args).await
    } else if sd_args.type_of_inference == "model" {
-        process_job_sd(&args).await
+        process_job_sd_upload(&args).await
    } else {
        Err(ProcessSingleJobError::Other(anyhow!("inference type doesn't exist!")))
    }
 }

-pub async fn process_job_sd(
-    args: &StableDiffusionProcessArgs<'_>
-) -> Result<JobSuccessResult, ProcessSingleJobError> {
-
-    let job = args.job;
-    let deps = args.job_dependencies;
-    let mysql_pool = &deps.db.mysql_pool;
-
-    let sd_args = sd_args_from_job(&args).await?;
-
-    let sd_deps = match
-        &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
-    {
-        None => {
-            return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
-        }
-        Some(val) => { val }
-    };
-
-    let creator_ip_address = &job.creator_ip_address;
-    let creator_user_token: UserToken;
-
-    match &job.maybe_creator_user_token {
-        Some(token) => {
-            creator_user_token = UserToken::new_from_str(token);
-        }
-        None => {
-            return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
-        }
-    }
-
-    // The parameters will be updated on another screen perhaps?
-    // so right now it will fill with the availible  values.
-
-    let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
-    let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
-        .new_tempdir(&work_temp_dir)
-        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
-
-    let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
-    //let vae_path = work_temp_dir.path().join("vae.safetensors");
-    let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
-    let output_path = work_temp_dir.path().join("output");
-
-    info!("Paths to download to:");
-    info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
-    info!("vae_path: {:?}", vae_path);
-    info!("output_path: {:?}", output_path);
-
-    let download_url = match sd_args.maybe_upload_path {
-        Some(val) => { val }
-        None => { "".to_string() }
-    };
-
-    if download_url.len() == 0 {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
-        );
-    }
-
-    let file_name = "sd_checkpoint.safetensors";
-    let download_script = easyenv::get_env_string_or_default(
-        "DOWNLOAD_SCRIPT",
-        "download_internet_file.py"
-    );
-
-    let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
-        None, None, None);
-
-    info!("Downloading {}", download_url);
-
-    let download_filename = match
-        google_drive_downloader.download_file_with_file_name(
-            &download_url,
-            &work_temp_dir,
-            file_name
-        ).await
-    {
-        Ok(filename) => filename,
-        Err(_e) => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download"))),
-    };
-
-    let download_file_path = work_temp_dir.path().join(download_filename);
-
-    if file_exists(download_file_path.as_path()) == false {
-        return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download SD Model from Google")));
-    }
-
-    info!("File Retrieved at {}", download_file_path.display());
-
-    // download vae for model
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
-        .await
-        .map_err(|err| {
-            error!("could not download VAE: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
-        })?;
-
-    info!("VAE downloaded to: {:?}", &vae_path);
-
-//    // use this vae doesn't matter though
-//    // VAE token for now
-//    let vae_token = String::from("REPLACE_ME");
-//    let model_weight_vae = ModelWeightToken(vae_token);
-//    let vae_weight_record = get_weight_by_token(
-//        &model_weight_vae,
-//        false,
-//        &deps.db.mysql_pool
-//    ).await?;
-//    let vae_weight_record = match vae_weight_record {
-//        Some(val) => val,
-//        None => {
-//            return Err(
-//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-//            );
-//        }
-//    };
-//
-//    let vae_details = RemoteCloudBucketDetails::new(
-//        vae_weight_record.public_bucket_hash.clone(),
-//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-//    );
-//
-//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
-
-    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-
-    let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
-    let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
-
-    // run inference on loRA downloaded
-    let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
-        work_dir: work_temp_dir.path().to_path_buf(),
-        output_file: output_path.clone(),
-        stderr_output_file: &stderr_output_file,
-        stdout_output_file: &stdout_output_file,
-        prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
-        negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
-        number_of_samples:20,
-        samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        lora_path: PathBuf::from(""),
-        checkpoint_path: sd_checkpoint_path.clone(),
-        vae: vae_path.clone(),
-        batch_count: sd_args.maybe_batch_count.unwrap_or(1),
-    });
-
-    if !exit_status.is_success() {
-        error!("SD inference failed: {:?}", exit_status);
-
-        let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            warn!("Captured stderr output: {}", contents);
-
-            //match categorize_error(&contents)  {
-            //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //    warn!("Face not detected in source image");
-            //    error = ProcessSingleJobError::FaceDetectionFailure;
-            //  }
-            //  _ => {}
-            //}
-        }
-        return Err(error);
-    }
-
-
-    // check if file exists if it does not error out....
-    let path = output_path.clone();
-    let file_path = format!("{}_{}.png", path_to_string(path), 0);
-
-    if !file_exists(file_path.path()) {
-        error!("Output file did not exist: {:?}", file_path);
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            error!("Captured stderr output: {}", contents);
-        }
-
-        return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model")));
-    }
-
-    // If it worked and didn't fail! then we should save and create the weight.
-    // upload and create weights for loRA...
-    let weights_sd_descriptor = Box::new(WeightsSD15Descriptor{});
-    let metadata = remote_cloud_file_client.upload_file(weights_sd_descriptor,sd_checkpoint_path.to_str().unwrap_or_default()).await?;
-    // chekc the model hash for duplicated models.
-    let bucket_details = match metadata.bucket_details {
-        Some(metadata) => metadata, 
-        None => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
-        }
-    };
-
-    let model_weight_token = &ModelWeightToken::generate();
-    let model_weight_token_result = create_weight(CreateModelWeightsArgs {
-        token: &model_weight_token,
-        weights_type: WeightsType::StableDiffusion15,
-        weights_category: WeightsCategory::ImageGeneration,
-        title: sd_args.maybe_name.unwrap_or(String::from("")),
-        maybe_description_markdown: sd_args.maybe_description,
-        maybe_description_rendered_html: None,
-        creator_user_token: Some(&creator_user_token),
-        creator_ip_address,
-        creator_set_visibility: Default::default(),
-        maybe_last_update_user_token: None,
-        original_download_url: Some(download_url),
-        original_filename: None,
-        file_size_bytes: metadata.file_size_bytes,
-        file_checksum_sha2: metadata.sha256_checksum,
-        public_bucket_hash: bucket_details.object_hash,
-        maybe_public_bucket_prefix: Some(bucket_details.prefix),
-        maybe_public_bucket_extension:Some(bucket_details.suffix),
-        version: sd_args.maybe_version.unwrap_or(0),
-        mysql_pool,
-    }).await?;
-
-    Ok(JobSuccessResult {
-        maybe_result_entity: Some(ResultEntity {
-            entity_type: InferenceResultType::UploadModel,
-            entity_token: model_weight_token_result.to_string(),
-        }),
-        inference_duration: Duration::from_secs(0),
-    })
-}
-
-pub async fn process_job_lora(
-    args: &StableDiffusionProcessArgs<'_>
-) -> Result<JobSuccessResult, ProcessSingleJobError> {
-
-    let job = args.job;
-    let deps = args.job_dependencies;
-    let mysql_pool = &deps.db.mysql_pool;
-
-    let sd_args = sd_args_from_job(&args).await?;
-
-    let sd_deps = match
-        &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
-    {
-        Some(val) => val,
-        None => return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies"))),
-    };
-
-    let creator_ip_address = &job.creator_ip_address;
-    let creator_user_token = match job.maybe_creator_user_token.as_deref() {
-        Some(token) => UserToken::new_from_str(token),
-        None => return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))),
-    };
-
-
-    let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
-    let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
-        .new_tempdir(&work_temp_dir)
-        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
-
-    let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
-    let lora_path = work_temp_dir.path().join("lora.safetensors"); // input path into execution
-    let vae_path = work_temp_dir.path().join("vae.safetensors");
-    let output_path = work_temp_dir.path().join("output");
-
-    info!("Paths to download to:");
-    info!("sd_checkpoint_path:{}", sd_checkpoint_path.display());
-    info!("lora_path:{}", lora_path.display());
-    info!("vae_path:{}", vae_path.display());
-    info!("output_path:{}", output_path.display());
-
-    let download_url = match sd_args.maybe_lora_upload_path {
-        Some(val) => { val }
-        None => { "".to_string() }
-    };
-
-    if download_url.len() == 0 {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
-        );
-    }
-
-    let file_name = "lora.safetensors";
-
-    let download_script = easyenv::get_env_string_or_default(
-        "DOWNLOAD_SCRIPT",
-        "download_internet_file.py"
-    );
-
-    // Download 
-    let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
-        None,
-        None, 
-        None);
-
-    info!("Downloading {}", download_url);
-
-    let download_filename = match
-        google_drive_downloader.download_file_with_file_name(
-            &download_url,
-            &work_temp_dir,
-            file_name
-        ).await
-    {
-        Ok(filename) => filename,
-        Err(_e) => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download")));
-        }
-    };
-
-    let download_file_path = work_temp_dir.path().join(download_filename);
-    info!("File Retrieved at {}", download_file_path.display());
-    if file_exists(download_file_path.as_path()) == false {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download loRA Model from Google Drive"))
-        );
-    }
-
-
-    // download model + vae for lora
-    // use lora with whatever checkpoint make it fixed id if it works submit it
-    let weight_token_sd_model_token = ModelWeightToken::new_from_str(&sd_deps.predefined_sd_weight_token);
-
-    info!("Using predefined SD weight token: {:?}", &weight_token_sd_model_token);
-
-    let sd_weight_record = get_weight_by_token(
-        &weight_token_sd_model_token,
-        false,
-        &deps.db.mysql_pool
-    ).await?;
-    let sd_weight_record = match sd_weight_record {
-        Some(val) => val,
-        None => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(anyhow!("No SD weight baked in for loRA upload inference check? thats a problem."))
-            );
-        }
-    };
-
-    let sd_weight_details = RemoteCloudBucketDetails::new(
-        sd_weight_record.public_bucket_hash.clone(),
-        sd_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-        sd_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-    );
-
-    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-    remote_cloud_file_client.download_file(sd_weight_details, path_to_string(sd_checkpoint_path.clone())).await?;
-
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
-        .await
-        .map_err(|err| {
-            error!("could not download VAE: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
-        })?;
-
-//    // use this vae doesn't matter though
-//    // VAE token for now
-//    let vae_token = String::from("REPLACE_ME");
-//    let model_weight_vae = ModelWeightToken(vae_token);
-//    let vae_weight_record = get_weight_by_token(
-//        &model_weight_vae,
-//        false,
-//        &deps.db.mysql_pool
-//    ).await?;
-//    let vae_weight_record = match vae_weight_record {
-//        Some(val) => val,
-//        None => {
-//            return Err(
-//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-//            );
-//        }
-//    };
-//
-//    let vae_details = RemoteCloudBucketDetails::new(
-//        vae_weight_record.public_bucket_hash.clone(),
-//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-//    );
-//
-//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
-
-    let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
-    let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
-
-    // run inference on loRA downloaded
-    let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
-        work_dir: work_temp_dir.path().to_path_buf(),
-        output_file: output_path.clone(),
-        stderr_output_file: &stderr_output_file,
-        stdout_output_file: &stdout_output_file,
-        prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
-        negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
-        number_of_samples:20,
-        samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        lora_path: lora_path.clone(),
-        checkpoint_path: sd_checkpoint_path.clone(),
-        vae: vae_path.clone(),
-        batch_count: sd_args.maybe_batch_count.unwrap_or(1),
-    });
-
-    if !exit_status.is_success() {
-        error!("SD inference failed: {:?}", exit_status);
-
-        let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            warn!("Captured stderr output: {}", contents);
-
-            //match categorize_error(&contents)  {
-            //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //    warn!("Face not detected in source image");
-            //    error = ProcessSingleJobError::FaceDetectionFailure;
-            //  }
-            //  _ => {}
-            //}
-        }
-        return Err(error);
-    }
-
-    // check if file exists if it does not error out....
-    let path = output_path.clone();
-    let file_path = format!("{}_{}.png", path_to_string(path), 0);
-    
-    if file_exists(file_path.path()) == false {
-        return Err(
-            ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a LoRA"))
-        );
-    }
-
-    // If it worked and didn't fail! then we should save and create the weight.
-    // upload and create weights for loRA...
-    let lora_descriptor = Box::new(WeightsLoRADescriptor{});
-    let metadata = remote_cloud_file_client.upload_file(lora_descriptor,lora_path.to_str().unwrap_or_default()).await?;
-
-    let bucket_details = match metadata.bucket_details {
-        Some(metadata) => metadata, 
-        None => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
-        }
-    };
-
-    let model_weight_token = &ModelWeightToken::generate();
-    let model_weight_token_result = create_weight(CreateModelWeightsArgs {
-        token: &model_weight_token,
-        weights_type: WeightsType::LoRA,
-        weights_category: WeightsCategory::ImageGeneration,
-        title: sd_args.maybe_name.unwrap_or(String::from("")),
-        maybe_description_markdown: sd_args.maybe_description,
-        maybe_description_rendered_html: None,
-        creator_user_token: Some(&creator_user_token),
-        creator_ip_address,
-        creator_set_visibility: Default::default(),
-        maybe_last_update_user_token: None,
-        original_download_url: Some(download_url),
-        original_filename: None,
-        file_size_bytes: metadata.file_size_bytes,
-        file_checksum_sha2: metadata.sha256_checksum,
-        public_bucket_hash: bucket_details.object_hash,
-        maybe_public_bucket_prefix: Some(bucket_details.prefix),
-        maybe_public_bucket_extension:Some(bucket_details.suffix),
-        version: sd_args.maybe_version.unwrap_or(0),
-        mysql_pool,
-    }).await?;
-
-    Ok(JobSuccessResult {
-        maybe_result_entity: Some(ResultEntity {
-            entity_type: InferenceResultType::UploadModel,
-            entity_token: model_weight_token_result.to_string(),
-        }),
-        inference_duration: Duration::from_secs(0),
-    })
-
-}
-
-
-pub async fn process_job_inference(
-    args: &StableDiffusionProcessArgs<'_>
-) -> Result<JobSuccessResult, ProcessSingleJobError> {
-    let job = args.job;
-    let deps = args.job_dependencies;
-    let mysql_pool = &deps.db.mysql_pool;
-
-    let sd_args = sd_args_from_job(&args).await?;
-    let sd_deps: &crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies = match
-        &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
-    {
-        None => {
-            return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
-        }
-        Some(val) => { val }
-    };
-
-    let _creator_user_token: UserToken;
-    let _anon_user_token: Option<AnonymousVisitorTrackingToken>;
-
-    match &job.maybe_creator_anonymous_visitor_token {
-        Some(token) => {
-            let anonymous_visitor_token = AnonymousVisitorTrackingToken::new_from_str(token);
-            _anon_user_token = Some(anonymous_visitor_token);
-        }
-        None => {
-            _anon_user_token = None;
-        }
-    }
-
-    match &job.maybe_creator_user_token {
-        Some(token) => {
-            _creator_user_token = UserToken::new_from_str(token);
-        }
-        None => {
-            return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
-        }
-    }
-
-    let _job_progress_reporter = args.job_dependencies.clients.job_progress_reporter
-        .new_generic_inference(job.inference_job_token.as_str())
-        .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
-
-    //==================== TEMP DIR ==================== //
-
-    let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
-
-    //NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
-    let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
-        .new_tempdir(&work_temp_dir)
-        .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
-
-    let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
-    let mut lora_path = work_temp_dir.path().join("lora.safetensors");
-    let vae_path = work_temp_dir.path().join("vae.safetensors");
-    let output_path = work_temp_dir.path().join("output");
-
-    info!("Paths to download to:");
-    info!("sd_checkpoint_path:{}", sd_checkpoint_path.display());
-    info!("lora_path:{}", lora_path.display());
-    info!("vae_path:{}", vae_path.display());
-    info!("output_path:{}", output_path.display());
-
-    // thread::sleep(seconds) to check the directory
-
-    // // Unpack loRA and Checkpoint
-    // // run inference by downloading from google drive.
-    let lora_token = sd_args.maybe_lora_model_token;
-    let weight_token = sd_args.maybe_sd_model_token.clone();
-
-    let retrieved_sd_record = match weight_token {
-        Some(ref token) => {
-            let retrieved_sd_record = get_weight_by_token(
-                &token,
-                false,
-                &deps.db.mysql_pool
-            ).await?;
-            match retrieved_sd_record {
-                Some(record) => record,
-                None => {
-                    return Err(
-                        ProcessSingleJobError::from_anyhow_error(anyhow!("no record of model!"))
-                    );
-                }
-            }
-        }
-        None => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))
-            );
-        }
-    };
-    // origin file name needs to be just the file name  /tmp/downloads_long_lived/temp_stable_diffusion_inference_32.8qJJljxWWZeD/output_0.png
-    // ignore if no lora token
-    let retrieved_loRA_record = match lora_token {
-        Some(token) => {
-            let retrieved_loRA_record = get_weight_by_token(
-                &token,
-                false,
-                &deps.db.mysql_pool
-            ).await?;
-
-            Some(retrieved_loRA_record)
-        }
-        None => None,
-    };
-
-    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
-    
-    // Details for SD checkpoint
-    let details = RemoteCloudBucketDetails::new(
-        retrieved_sd_record.public_bucket_hash.clone(),
-        retrieved_sd_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-        retrieved_sd_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-    );
-
-    remote_cloud_file_client.download_file(
-        details,
-        path_to_string(sd_checkpoint_path.clone())
-    ).await?;
-
-    let mut lora_name = String::from("");
-    let mut lora_token = String::from("");
-
-    match retrieved_loRA_record {
-        Some(record) => {
-            match record {
-                Some(model_weight_record) => {
-                    let lora_details = RemoteCloudBucketDetails::new(
-                        model_weight_record.public_bucket_hash.clone(),
-                        model_weight_record.maybe_public_bucket_prefix
-                            .clone()
-                            .unwrap_or_else(|| "".to_string()),
-                        model_weight_record.maybe_public_bucket_extension
-                            .clone()
-                            .unwrap_or_else(|| "".to_string())
-                    );
-
-                    lora_name = model_weight_record.title;
-                    lora_token = model_weight_record.token.to_string();
-                    remote_cloud_file_client.download_file(
-                        lora_details,
-                        path_to_string(lora_path.clone())
-                    ).await?;
-                }
-                None => {
-                    lora_path.clear();
-                }
-            }
-        }
-        None => {
-            lora_path.clear();
-        }
-    }
-
-    args.job_dependencies
-        .buckets
-        .public_bucket_client
-        .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
-        .await
-        .map_err(|err| {
-            error!("could not download VAE: {:?}", err);
-            ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
-        })?;
-
-//    // VAE token for now
-//    let vae_token = String::from("REPLACE_ME");
-//    let model_weight_vae = ModelWeightToken(vae_token);
-//
-//    let vae_weight_record = get_weight_by_token(
-//        &model_weight_vae,
-//        false,
-//        &deps.db.mysql_pool
-//    ).await?;
-//
-//    let vae_weight_record = match vae_weight_record {
-//        Some(val) => val,
-//        None => {
-//            return Err(
-//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-//            );
-//        }
-//    };
-//
-//    let vae_details = RemoteCloudBucketDetails::new(
-//        vae_weight_record.public_bucket_hash.clone(),
-//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
-//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
-//    );
-//
-//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
-
-    let prompt = match sd_args.maybe_prompt {
-        Some(val) => val,
-        None => {
-            return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No Prompt provided!")));
-        }
-    };
-
-    let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
-    let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
-
-    let number_of_samples = match sd_args.maybe_number_of_samples {
-        Some(val) => val,
-        None => 20,
-    };
-
-    let inference_start_time = Instant::now();
-
-    let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
-        work_dir: work_temp_dir.path().to_path_buf(),
-        output_file: output_path.clone(),
-        stderr_output_file: &stderr_output_file,
-        stdout_output_file: &stdout_output_file,
-        prompt: prompt.clone(),
-        negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
-        number_of_samples,
-        samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        lora_path: lora_path.clone(),
-        checkpoint_path: sd_checkpoint_path.clone(),
-        vae: vae_path.clone(),
-        batch_count: sd_args.maybe_batch_count.unwrap_or(1),
-    });
-
-    if !exit_status.is_success() {
-        error!("SD inference failed: {:?}", exit_status);
-
-        let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
-
-        if let Ok(contents) = read_to_string(&stderr_output_file) {
-            warn!("Captured stderr output: {}", contents);
-
-            //match categorize_error(&contents)  {
-            //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //    warn!("Face not detected in source image");
-            //    error = ProcessSingleJobError::FaceDetectionFailure;
-            //  }
-            //  _ => {}
-            //}
-        }
-        return Err(error);
-    }
-
-
-    // hack to check the directory before clean up.
-    //   let thirtyMinutes = 1800;
-    //   thread::sleep(Duration::from_secs(thirtyMinutes));
-      // upload media and create a record.
-
-    let inference_duration = Instant::now().duration_since(inference_start_time);
-
-    // run a for loop for output images output_0 in the folder then use upload media.
-    // pngs
-
-    let mut entries = vec![];
-
-    let inputs = InferenceValues {
-        prompt: prompt.clone(),
-        cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
-        negative_prompt: sd_args.maybe_n_prompt,
-        lora_model_weight_token: Some(lora_token),
-        lora_name: Some(lora_name),
-        sampler: sd_args.maybe_sampler.unwrap_or(String::from("Euler a")),
-        width: sd_args.maybe_width.unwrap_or(512),
-        height: sd_args.maybe_height.unwrap_or(512),
-        seed: sd_args.maybe_seed.unwrap_or(1),
-        number_of_samples,
-    };
-
-    let inputs = match serde_json::to_string(&inputs) {
-        Ok(result) => result,
-        Err(_err) => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(anyhow!("couldn't serialize metadata."))
-            );
-        }
-    };
-
-
-    for i in 0..sd_args.maybe_batch_count.unwrap_or(1) {
-        let path = output_path.clone();
-
-        let file_path = format!("{}_{}.png", path_to_string(path), i);
-
-        println!("Upload File Path:{}", file_path);
-
-        let metadata = remote_cloud_file_client.upload_file(
-            Box::new(MediaImagePngDescriptor {}),
-            file_path.as_ref()
-        ).await?;
-
-        let bucket_details = match metadata.bucket_details {
-            Some(val) => { val }
-            None => {
-                return Err(
-                    ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
-                );
-            }
-        };
-
-        // extra_file_modification_info: todo!(), // JSON ENCODED STRUCT
-        let media_file_token = insert_media_file_generic(InsertArgs {
-            pool: mysql_pool,
-            job,
-            media_type: MediaFileType::Image,
-            origin_category: MediaFileOriginCategory::Upload,
-            origin_product_category: MediaFileOriginProductCategory::ImageGeneration,
-            maybe_origin_model_type: Some(MediaFileOriginModelType::StableDiffusion15),
-            maybe_origin_model_token: weight_token.clone(),
-            maybe_origin_filename: Some(file_path),
-            is_batch_generated: true,
-            maybe_mime_type: Some(metadata.mimetype.as_ref()),
-            file_size_bytes: metadata.file_size_bytes,
-            maybe_duration_millis: Some(inference_duration.as_millis() as u64),
-            maybe_audio_encoding: None,
-            maybe_video_encoding: None,
-            maybe_frame_width: Some(sd_args.maybe_width.unwrap_or(512)),
-            maybe_frame_height: Some(sd_args.maybe_height.unwrap_or(512)),
-            checksum_sha2: metadata.sha256_checksum.as_str(),
-            public_bucket_directory_hash: bucket_details.object_hash.as_str(),
-            maybe_public_bucket_prefix: Some(bucket_details.prefix.as_str()),
-            maybe_public_bucket_extension: Some(bucket_details.suffix.as_str()),
-            extra_file_modification_info: Some(&inputs),
-            maybe_creator_file_synthetic_id_category: IdCategory::MediaFile,
-            maybe_creator_category_synthetic_id_category: IdCategory::ModelWeights,
-            maybe_mod_user_token: None,
-            is_generated_on_prem: args.job_dependencies.job.info.container.is_on_prem,
-            generated_by_worker: Some(&args.job_dependencies.job.info.container.hostname),
-            generated_by_cluster: Some(&args.job_dependencies.job.info.container.cluster_name),
-        }).await?;
-
-        let batch_generation_entity: BatchGenerationEntity = BatchGenerationEntity::MediaFile(
-            media_file_token.0
-        );
-        entries.push(batch_generation_entity);
-    }
-
-    let mut transaction = mysql_pool.begin().await.unwrap();
-    
-    let batch_token_result = insert_batch_generation_records(InsertBatchArgs {
-        entries,
-        transaction: &mut transaction,
-    }).await;
-
-    let batch_token = match batch_token_result {
-        Ok(v) => { v.to_string() }
-        Err(_err) => {
-            return Err(
-                ProcessSingleJobError::from_anyhow_error(
-                    anyhow!("No batch token? something has failed.")
-                )
-            );
-        }
-    };
-
-    Ok(JobSuccessResult { // TODO: batch token needs to go here
-        maybe_result_entity: Some(ResultEntity {
-            entity_type: InferenceResultType::MediaFile,
-            entity_token: batch_token,
-        }),
-        inference_duration,
-    })
-}

 #[ignore]
 #[cfg(test)]
@@ -0,0 +1,414 @@
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use actix_web::dev::ResourcePath;
+use anyhow::anyhow;
+use log::{error, info, warn};
+use serde_json;
+
+use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
+use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
+use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
+use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use filesys::file_exists::file_exists;
+use filesys::path_to_string::path_to_string;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::{insert_batch_generation_records, InsertBatchArgs};
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::{insert_media_file_generic, InsertArgs};
+use mysql_queries::queries::model_weights::create::create_weight::{
+  create_weight,
+  CreateModelWeightsArgs,
+};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::image_generation::sd::process_job::{InferenceValues, sd_args_from_job, StableDiffusionProcessArgs};
+use crate::job::job_types::image_generation::sd::process_job_lora_upload::process_job_lora_upload;
+use crate::job::job_types::image_generation::sd::process_job_sd_upload::process_job_sd_upload;
+use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job_dependencies::JobDependencies;
+
+pub async fn process_job_inference(
+  args: &StableDiffusionProcessArgs<'_>
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+  let job = args.job;
+  let deps = args.job_dependencies;
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let sd_args = sd_args_from_job(&args).await?;
+  let sd_deps: &crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies = match
+  &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
+  {
+    None => {
+      return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
+    }
+    Some(val) => { val }
+  };
+
+  let _creator_user_token: UserToken;
+  let _anon_user_token: Option<AnonymousVisitorTrackingToken>;
+
+  match &job.maybe_creator_anonymous_visitor_token {
+    Some(token) => {
+      let anonymous_visitor_token = AnonymousVisitorTrackingToken::new_from_str(token);
+      _anon_user_token = Some(anonymous_visitor_token);
+    }
+    None => {
+      _anon_user_token = None;
+    }
+  }
+
+  match &job.maybe_creator_user_token {
+    Some(token) => {
+      _creator_user_token = UserToken::new_from_str(token);
+    }
+    None => {
+      return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
+    }
+  }
+
+  let _job_progress_reporter = args.job_dependencies.clients.job_progress_reporter
+      .new_generic_inference(job.inference_job_token.as_str())
+      .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
+
+  //==================== TEMP DIR ==================== //
+
+  let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
+
+  //NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
+  let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  //let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
+  let mut lora_path = work_temp_dir.path().join("lora.safetensors");
+  //let vae_path = work_temp_dir.path().join("vae.safetensors");
+  let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
+  let output_path = work_temp_dir.path().join("output");
+
+  info!("Paths to download to:");
+  info!("lora_path: {:?}", lora_path);
+  info!("vae_path: {:?}", vae_path);
+  info!("output_path: {:?}", output_path);
+
+  // thread::sleep(seconds) to check the directory
+
+  // // Unpack loRA and Checkpoint
+  // // run inference by downloading from google drive.
+  let lora_token = sd_args.maybe_lora_model_token;
+///  let weight_token = sd_args.maybe_sd_model_token.clone();
+
+  let sd_model_weight_token = match sd_args.maybe_sd_model_token {
+    None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))),
+    Some(ref token) => token,
+  };
+
+  let sd_model_weight = get_weight_by_token(
+    sd_model_weight_token,
+    false,
+    &deps.db.mysql_pool
+  ).await?;
+
+  let sd_model_weight = match sd_model_weight {
+    None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no record of sd model!"))),
+    Some(record) => record,
+  };
+
+  let sd_checkpoint_path = args.job_dependencies
+      .fs
+      .model_weights_cache_directory
+      .get_model_weight_from_cache_or_bucket(&sd_model_weight)
+      .await?;
+
+  info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
+
+//  let retrieved_sd_record = match sd_args.maybe_sd_model_token {
+//    None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))),
+//    Some(ref token) => {
+//      let retrieved_sd_record = get_weight_by_token(
+//        &token,
+//        false,
+//        &deps.db.mysql_pool
+//      ).await?;
+//      match retrieved_sd_record {
+//        Some(record) => record,
+//        None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no record of model!"))),
+//      }
+//    }
+//  };
+
+  // origin file name needs to be just the file name  /tmp/downloads_long_lived/temp_stable_diffusion_inference_32.8qJJljxWWZeD/output_0.png
+  // ignore if no lora token
+
+  let mut maybe_lora_record = None;
+
+  if let Some(token) = lora_token {
+    maybe_lora_record = get_weight_by_token(
+      &token,
+      false,
+      &deps.db.mysql_pool
+    ).await?;
+  }
+
+  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+
+///  // Details for SD checkpoint
+///  let details = RemoteCloudBucketDetails::new(
+///    retrieved_sd_record.public_bucket_hash.clone(),
+///    retrieved_sd_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+///    retrieved_sd_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+///  );
+///
+///  remote_cloud_file_client.download_file(
+///    details,
+///    path_to_string(sd_checkpoint_path.clone())
+///  ).await?;
+
+  let mut lora_name = String::from("");
+  let mut lora_token = String::from("");
+
+  match maybe_lora_record {
+    None => {
+      lora_path.clear();
+    }
+    Some(lora_record) => {
+/////      let lora_details = RemoteCloudBucketDetails::new(
+/////        lora_record.public_bucket_hash.clone(),
+/////        lora_record.maybe_public_bucket_prefix
+/////            .clone()
+/////            .unwrap_or_else(|| "".to_string()),
+/////        lora_record.maybe_public_bucket_extension
+/////            .clone()
+/////            .unwrap_or_else(|| "".to_string())
+/////      );
+/////
+/////
+/////      remote_cloud_file_client.download_file(
+/////        lora_details,
+/////        path_to_string(lora_path.clone())
+/////      ).await?;
+
+      lora_name = lora_record.title.to_string();
+      lora_token = lora_record.token.to_string();
+
+      lora_path = args.job_dependencies
+          .fs
+          .model_weights_cache_directory
+          .get_model_weight_from_cache_or_bucket(&lora_record)
+          .await?;
+    }
+  }
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
+      .await
+      .map_err(|err| {
+        error!("could not download VAE: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
+      })?;
+
+  let prompt = match sd_args.maybe_prompt {
+    Some(val) => val,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No Prompt provided!")));
+    }
+  };
+
+  let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
+  let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
+
+  let number_of_samples = match sd_args.maybe_number_of_samples {
+    Some(val) => val,
+    None => 20,
+  };
+
+  let inference_start_time = Instant::now();
+
+  let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
+    work_dir: work_temp_dir.path().to_path_buf(),
+    output_file: output_path.clone(),
+    stderr_output_file: &stderr_output_file,
+    stdout_output_file: &stdout_output_file,
+    prompt: prompt.clone(),
+    negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
+    number_of_samples,
+    samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    lora_path: lora_path.clone(),
+    checkpoint_path: sd_checkpoint_path.clone(),
+    vae: vae_path.clone(),
+    batch_count: sd_args.maybe_batch_count.unwrap_or(1),
+  });
+
+  if !exit_status.is_success() {
+    error!("SD inference failed: {:?}", exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+    return Err(error);
+  }
+
+
+  // hack to check the directory before clean up.
+  //   let thirtyMinutes = 1800;
+  //   thread::sleep(Duration::from_secs(thirtyMinutes));
+  // upload media and create a record.
+
+  let inference_duration = Instant::now().duration_since(inference_start_time);
+
+  // run a for loop for output images output_0 in the folder then use upload media.
+  // pngs
+
+  let mut entries = vec![];
+
+  let inputs = InferenceValues {
+    prompt: prompt.clone(),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    negative_prompt: sd_args.maybe_n_prompt,
+    lora_model_weight_token: Some(lora_token),
+    lora_name: Some(lora_name),
+    sampler: sd_args.maybe_sampler.unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    number_of_samples,
+  };
+
+  let inputs = match serde_json::to_string(&inputs) {
+    Ok(result) => result,
+    Err(_err) => {
+      return Err(
+        ProcessSingleJobError::from_anyhow_error(anyhow!("couldn't serialize metadata."))
+      );
+    }
+  };
+
+  let mut maybe_first_media_file_token = None;
+
+  for i in 0..sd_args.maybe_batch_count.unwrap_or(1) {
+    let path = output_path.clone();
+
+    let file_path = format!("{}_{}.png", path_to_string(path), i);
+
+    println!("Upload File Path:{}", file_path);
+
+    let metadata = remote_cloud_file_client.upload_file(
+      Box::new(MediaImagePngDescriptor {}),
+      file_path.as_ref()
+    ).await?;
+
+    let bucket_details = match metadata.bucket_details {
+      Some(val) => { val }
+      None => {
+        return Err(
+          ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
+        );
+      }
+    };
+
+
+    // extra_file_modification_info: todo!(), // JSON ENCODED STRUCT
+    let (media_file_token, _id) = insert_media_file_generic(InsertArgs {
+      pool: mysql_pool,
+      job,
+      media_type: MediaFileType::Image,
+      origin_category: MediaFileOriginCategory::Upload,
+      origin_product_category: MediaFileOriginProductCategory::ImageGeneration,
+      maybe_origin_model_type: Some(MediaFileOriginModelType::StableDiffusion15),
+      maybe_origin_model_token: Some(sd_model_weight_token.clone()),
+      maybe_origin_filename: Some(file_path),
+      is_batch_generated: true,
+      maybe_mime_type: Some(metadata.mimetype.as_ref()),
+      file_size_bytes: metadata.file_size_bytes,
+      maybe_duration_millis: Some(inference_duration.as_millis() as u64),
+      maybe_audio_encoding: None,
+      maybe_video_encoding: None,
+      maybe_frame_width: Some(sd_args.maybe_width.unwrap_or(512)),
+      maybe_frame_height: Some(sd_args.maybe_height.unwrap_or(512)),
+      checksum_sha2: metadata.sha256_checksum.as_str(),
+      public_bucket_directory_hash: bucket_details.object_hash.as_str(),
+      maybe_public_bucket_prefix: Some(bucket_details.prefix.as_str()),
+      maybe_public_bucket_extension: Some(bucket_details.suffix.as_str()),
+      extra_file_modification_info: Some(&inputs),
+      maybe_creator_file_synthetic_id_category: IdCategory::MediaFile,
+      maybe_creator_category_synthetic_id_category: IdCategory::ModelWeights,
+      maybe_mod_user_token: None,
+      is_generated_on_prem: args.job_dependencies.job.info.container.is_on_prem,
+      generated_by_worker: Some(&args.job_dependencies.job.info.container.hostname),
+      generated_by_cluster: Some(&args.job_dependencies.job.info.container.cluster_name),
+    }).await?;
+
+    if maybe_first_media_file_token.is_none() {
+      maybe_first_media_file_token = Some(media_file_token.clone());
+    }
+
+    let batch_generation_entity: BatchGenerationEntity = BatchGenerationEntity::MediaFile(
+      media_file_token
+    );
+
+    entries.push(batch_generation_entity);
+  }
+
+  let mut transaction = mysql_pool.begin().await.unwrap();
+
+  let batch_token_result = insert_batch_generation_records(InsertBatchArgs {
+    entries,
+    transaction: &mut transaction,
+  }).await;
+
+  let batch_token = match batch_token_result {
+    Ok(v) => { v.to_string() }
+    Err(_err) => {
+      return Err(
+        ProcessSingleJobError::from_anyhow_error(
+          anyhow!("No batch token? something has failed.")
+        )
+      );
+    }
+  };
+
+  // TODO(bt,2024-02-12): Return the batch token instead. (We're not ready for that.)
+  Ok(JobSuccessResult {
+    inference_duration,
+    maybe_result_entity: maybe_first_media_file_token
+        .map(|media_file_token| {
+          ResultEntity {
+            entity_type: InferenceResultType::MediaFile,
+            entity_token: media_file_token.to_string(),
+          }
+        }),
+  })
+}
@@ -0,0 +1,318 @@
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use actix_web::dev::ResourcePath;
+use anyhow::anyhow;
+use log::{error, info, warn};
+use serde_json;
+
+use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
+use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
+use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
+use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use filesys::file_exists::file_exists;
+use filesys::path_to_string::path_to_string;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::insert_batch_generation_records;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::InsertBatchArgs;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::insert_media_file_generic;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::InsertArgs;
+use mysql_queries::queries::model_weights::create::create_weight::{
+  create_weight,
+  CreateModelWeightsArgs,
+};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::image_generation::sd::process_job::{sd_args_from_job, StableDiffusionProcessArgs};
+use crate::job::job_types::image_generation::sd::process_job_sd_upload::process_job_sd_upload;
+use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job_dependencies::JobDependencies;
+
+pub async fn process_job_lora_upload(
+  args: &StableDiffusionProcessArgs<'_>
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let job = args.job;
+  let deps = args.job_dependencies;
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let sd_args = sd_args_from_job(&args).await?;
+
+  let sd_deps = match
+  &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
+  {
+    Some(val) => val,
+    None => return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies"))),
+  };
+
+  let creator_ip_address = &job.creator_ip_address;
+  let creator_user_token = match job.maybe_creator_user_token.as_deref() {
+    Some(token) => UserToken::new_from_str(token),
+    None => return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token"))),
+  };
+
+
+  let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
+  let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
+  let lora_path = work_temp_dir.path().join("lora.safetensors"); // input path into execution
+  //let vae_path = work_temp_dir.path().join("vae.safetensors");
+  let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
+  let output_path = work_temp_dir.path().join("output");
+
+  info!("Paths to download to:");
+  info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
+  info!("lora_path: {:?}", lora_path);
+  info!("vae_path: {:?}", vae_path);
+  info!("output_path: {:?}", output_path);
+
+  let download_url = match sd_args.maybe_lora_upload_path {
+    Some(val) => { val }
+    None => { "".to_string() }
+  };
+
+  if download_url.len() == 0 {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
+    );
+  }
+
+  let file_name = "lora.safetensors";
+
+  let download_script = easyenv::get_env_string_or_default(
+    "DOWNLOAD_SCRIPT",
+    "download_internet_file.py"
+  );
+
+  // Download
+  let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
+                                                                None,
+                                                                None,
+                                                                None);
+
+  info!("Downloading {}", download_url);
+
+  let download_filename = match
+  google_drive_downloader.download_file_with_file_name(
+    &download_url,
+    &work_temp_dir,
+    file_name
+  ).await
+  {
+    Ok(filename) => filename,
+    Err(_e) => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download")));
+    }
+  };
+
+  let download_file_path = work_temp_dir.path().join(download_filename);
+  info!("File Retrieved at {}", download_file_path.display());
+  if file_exists(download_file_path.as_path()) == false {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download loRA Model from Google Drive"))
+    );
+  }
+
+
+//  // download model + vae for lora
+//  // use lora with whatever checkpoint make it fixed id if it works submit it
+//  let weight_token_sd_model_token = ModelWeightToken::new_from_str(&sd_deps.predefined_sd_weight_token);
+//
+//  info!("Using predefined SD weight token: {:?}", &weight_token_sd_model_token);
+//
+//  let sd_weight_record = get_weight_by_token(
+//    &weight_token_sd_model_token,
+//    false,
+//    &deps.db.mysql_pool
+//  ).await?;
+//  let sd_weight_record = match sd_weight_record {
+//    Some(val) => val,
+//    None => {
+//      return Err(
+//        ProcessSingleJobError::from_anyhow_error(anyhow!("No SD weight baked in for loRA upload inference check? thats a problem."))
+//      );
+//    }
+//  };
+//
+//  let sd_weight_details = RemoteCloudBucketDetails::new(
+//    sd_weight_record.public_bucket_hash.clone(),
+//    sd_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+//    sd_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+//  );
+//
+//  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+//  remote_cloud_file_client.download_file(sd_weight_details, path_to_string(sd_checkpoint_path.clone())).await?;
+
+  info!("Downloading predefined SD weight from: {:?} to {:?}",
+    &sd_deps.predefined_sd_weight_bucket_path,
+    &sd_checkpoint_path);
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.predefined_sd_weight_bucket_path, &sd_checkpoint_path)
+      .await
+      .map_err(|err| {
+        error!("could not download predefined SD weight: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download predefined SD weight: {:?}", err))
+      })?;
+
+  info!("Downloading predefined VAE weight from: {:?} to {:?}",
+    &sd_deps.vae_bucket_path,
+    &vae_path);
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
+      .await
+      .map_err(|err| {
+        error!("could not download VAE: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
+      })?;
+
+//    // use this vae doesn't matter though
+//    // VAE token for now
+//    let vae_token = String::from("REPLACE_ME");
+//    let model_weight_vae = ModelWeightToken(vae_token);
+//    let vae_weight_record = get_weight_by_token(
+//        &model_weight_vae,
+//        false,
+//        &deps.db.mysql_pool
+//    ).await?;
+//    let vae_weight_record = match vae_weight_record {
+//        Some(val) => val,
+//        None => {
+//            return Err(
+//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
+//            );
+//        }
+//    };
+//
+//    let vae_details = RemoteCloudBucketDetails::new(
+//        vae_weight_record.public_bucket_hash.clone(),
+//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+//    );
+//
+//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
+
+  let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
+  let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
+
+  // run inference on loRA downloaded
+  let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
+    work_dir: work_temp_dir.path().to_path_buf(),
+    output_file: output_path.clone(),
+    stderr_output_file: &stderr_output_file,
+    stdout_output_file: &stdout_output_file,
+    prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
+    negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
+    number_of_samples:20,
+    samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    lora_path: lora_path.clone(),
+    checkpoint_path: sd_checkpoint_path.clone(),
+    vae: vae_path.clone(),
+    batch_count: sd_args.maybe_batch_count.unwrap_or(1),
+  });
+
+  if !exit_status.is_success() {
+    error!("SD inference failed: {:?}", exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+    return Err(error);
+  }
+
+  // check if file exists if it does not error out....
+  let path = output_path.clone();
+  let file_path = format!("{}_{}.png", path_to_string(path), 0);
+
+  if file_exists(file_path.path()) == false {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a LoRA"))
+    );
+  }
+
+  // If it worked and didn't fail! then we should save and create the weight.
+  // upload and create weights for loRA...
+  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+  let lora_descriptor = Box::new(WeightsLoRADescriptor{});
+  let metadata = remote_cloud_file_client.upload_file(lora_descriptor,lora_path.to_str().unwrap_or_default()).await?;
+
+  let bucket_details = match metadata.bucket_details {
+    Some(metadata) => metadata,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
+    }
+  };
+
+  let model_weight_token = &ModelWeightToken::generate();
+  let model_weight_token_result = create_weight(CreateModelWeightsArgs {
+    token: &model_weight_token,
+    weights_type: WeightsType::LoRA,
+    weights_category: WeightsCategory::ImageGeneration,
+    title: sd_args.maybe_name.unwrap_or(String::from("")),
+    maybe_description_markdown: sd_args.maybe_description,
+    maybe_description_rendered_html: None,
+    creator_user_token: Some(&creator_user_token),
+    creator_ip_address,
+    creator_set_visibility: Default::default(),
+    maybe_last_update_user_token: None,
+    original_download_url: Some(download_url),
+    original_filename: None,
+    file_size_bytes: metadata.file_size_bytes,
+    file_checksum_sha2: metadata.sha256_checksum,
+    public_bucket_hash: bucket_details.object_hash,
+    maybe_public_bucket_prefix: Some(bucket_details.prefix),
+    maybe_public_bucket_extension:Some(bucket_details.suffix),
+    version: sd_args.maybe_version.unwrap_or(0),
+    mysql_pool,
+  }).await?;
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::UploadModel,
+      entity_token: model_weight_token_result.to_string(),
+    }),
+    inference_duration: Duration::from_secs(0),
+  })
+
+}
@@ -0,0 +1,280 @@
+use std::fs::read_to_string;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use actix_web::dev::ResourcePath;
+use anyhow::anyhow;
+use log::{error, info, warn};
+use serde_json;
+
+use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
+use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
+use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
+use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
+use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
+use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use filesys::file_exists::file_exists;
+use filesys::path_to_string::path_to_string;
+use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::insert_batch_generation_records;
+use mysql_queries::queries::batch_generations::insert_batch_generation_records::InsertBatchArgs;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::insert_media_file_generic;
+use mysql_queries::queries::media_files::create::insert_media_file_generic::InsertArgs;
+use mysql_queries::queries::model_weights::create::create_weight::{
+  create_weight,
+  CreateModelWeightsArgs,
+};
+use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
+use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::image_generation::sd::process_job::{sd_args_from_job, StableDiffusionProcessArgs};
+use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
+use crate::job_dependencies::JobDependencies;
+
+pub async fn process_job_sd_upload(
+  args: &StableDiffusionProcessArgs<'_>
+) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let job = args.job;
+  let deps = args.job_dependencies;
+  let mysql_pool = &deps.db.mysql_pool;
+
+  let sd_args = sd_args_from_job(&args).await?;
+
+  let sd_deps = match
+  &args.job_dependencies.job.job_specific_dependencies.maybe_stable_diffusion_dependencies
+  {
+    None => {
+      return Err(ProcessSingleJobError::Other(anyhow!("Missing Job Specific Dependencies")));
+    }
+    Some(val) => { val }
+  };
+
+  let creator_ip_address = &job.creator_ip_address;
+  let creator_user_token: UserToken;
+
+  match &job.maybe_creator_user_token {
+    Some(token) => {
+      creator_user_token = UserToken::new_from_str(token);
+    }
+    None => {
+      return Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Creator User Token")));
+    }
+  }
+
+  // The parameters will be updated on another screen perhaps?
+  // so right now it will fill with the availible  values.
+
+  let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
+  let work_temp_dir = args.job_dependencies.fs.scoped_temp_dir_creator_for_work
+      .new_tempdir(&work_temp_dir)
+      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;
+
+  let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors");
+  //let vae_path = work_temp_dir.path().join("vae.safetensors");
+  let vae_path = work_temp_dir.path().join("vae.pt"); // TODO: Should this be `.safetensors` or `.pt`?
+  let output_path = work_temp_dir.path().join("output");
+
+  info!("Paths to download to:");
+  info!("sd_checkpoint_path: {:?}", sd_checkpoint_path);
+  info!("vae_path: {:?}", vae_path);
+  info!("output_path: {:?}", output_path);
+
+  let download_url = match sd_args.maybe_upload_path {
+    Some(val) => { val }
+    None => { "".to_string() }
+  };
+
+  if download_url.len() == 0 {
+    return Err(
+      ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing"))
+    );
+  }
+
+  let file_name = "sd_checkpoint.safetensors";
+  let download_script = easyenv::get_env_string_or_default(
+    "DOWNLOAD_SCRIPT",
+    "download_internet_file.py"
+  );
+
+  let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script,
+                                                                None, None, None);
+
+  info!("Downloading {}", download_url);
+
+  let download_filename = match
+  google_drive_downloader.download_file_with_file_name(
+    &download_url,
+    &work_temp_dir,
+    file_name
+  ).await
+  {
+    Ok(filename) => filename,
+    Err(_e) => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download"))),
+  };
+
+  let download_file_path = work_temp_dir.path().join(download_filename);
+
+  if file_exists(download_file_path.as_path()) == false {
+    return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download SD Model from Google")));
+  }
+
+  info!("File Retrieved at {}", download_file_path.display());
+
+  // download vae for model
+
+  args.job_dependencies
+      .buckets
+      .public_bucket_client
+      .download_file_to_disk(&sd_deps.vae_bucket_path, &vae_path)
+      .await
+      .map_err(|err| {
+        error!("could not download VAE: {:?}", err);
+        ProcessSingleJobError::from_anyhow_error(anyhow!("could not download VAE: {:?}", err))
+      })?;
+
+  info!("VAE downloaded to: {:?}", &vae_path);
+
+//    // use this vae doesn't matter though
+//    // VAE token for now
+//    let vae_token = String::from("REPLACE_ME");
+//    let model_weight_vae = ModelWeightToken(vae_token);
+//    let vae_weight_record = get_weight_by_token(
+//        &model_weight_vae,
+//        false,
+//        &deps.db.mysql_pool
+//    ).await?;
+//    let vae_weight_record = match vae_weight_record {
+//        Some(val) => val,
+//        None => {
+//            return Err(
+//                ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem."))
+//            );
+//        }
+//    };
+//
+//    let vae_details = RemoteCloudBucketDetails::new(
+//        vae_weight_record.public_bucket_hash.clone(),
+//        vae_weight_record.maybe_public_bucket_prefix.clone().unwrap_or_else(|| "".to_string()),
+//        vae_weight_record.maybe_public_bucket_extension.clone().unwrap_or_else(|| "".to_string())
+//    );
+//
+//    let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+//    remote_cloud_file_client.download_file(vae_details, path_to_string(vae_path.clone())).await?;
+
+  let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await?;
+
+  let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
+  let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
+
+  // run inference on loRA downloaded
+  let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
+    work_dir: work_temp_dir.path().to_path_buf(),
+    output_file: output_path.clone(),
+    stderr_output_file: &stderr_output_file,
+    stdout_output_file: &stdout_output_file,
+    prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
+    negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
+    number_of_samples:20,
+    samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")),
+    width: sd_args.maybe_width.unwrap_or(512),
+    height: sd_args.maybe_height.unwrap_or(512),
+    cfg_scale: sd_args.maybe_cfg_scale.unwrap_or(7),
+    seed: sd_args.maybe_seed.unwrap_or(1),
+    lora_path: PathBuf::from(""),
+    checkpoint_path: sd_checkpoint_path.clone(),
+    vae: vae_path.clone(),
+    batch_count: sd_args.maybe_batch_count.unwrap_or(1),
+  });
+
+  if !exit_status.is_success() {
+    error!("SD inference failed: {:?}", exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+    return Err(error);
+  }
+
+  // check if file exists if it does not error out....
+  let path = output_path.clone();
+  let file_path = format!("{}_{}.png", path_to_string(path), 0);
+
+  if !file_exists(file_path.path()) {
+    error!("Output file did not exist: {:?}", file_path);
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      error!("Captured stderr output: {}", contents);
+    }
+
+    return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model")));
+  }
+
+  // If it worked and didn't fail! then we should save and create the weight.
+  // upload and create weights for loRA...
+  let weights_sd_descriptor = Box::new(WeightsSD15Descriptor{});
+  let metadata = remote_cloud_file_client.upload_file(weights_sd_descriptor,sd_checkpoint_path.to_str().unwrap_or_default()).await?;
+  // chekc the model hash for duplicated models.
+  let bucket_details = match metadata.bucket_details {
+    Some(metadata) => metadata,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to generate bucket details!")));
+    }
+  };
+
+  let model_weight_token: &ModelWeightToken = &ModelWeightToken::generate();
+  let model_weight_token_result = create_weight(CreateModelWeightsArgs {
+    token: &model_weight_token,
+    weights_type: WeightsType::StableDiffusion15,
+    weights_category: WeightsCategory::ImageGeneration,
+    title: sd_args.maybe_name.unwrap_or(String::from("")),
+    maybe_description_markdown: sd_args.maybe_description,
+    maybe_description_rendered_html: None,
+    creator_user_token: Some(&creator_user_token),
+    creator_ip_address,
+    creator_set_visibility: Default::default(),
+    maybe_last_update_user_token: None,
+    original_download_url: Some(download_url),
+    original_filename: None,
+    // file_size_bytes: metadata.file_size_bytes, // TODO(bt,2024-02-03): We need to migrate the column to be BIGINT
+    file_size_bytes: 0,
+    file_checksum_sha2: metadata.sha256_checksum,
+    public_bucket_hash: bucket_details.object_hash,
+    maybe_public_bucket_prefix: Some(bucket_details.prefix),
+    maybe_public_bucket_extension:Some(bucket_details.suffix),
+    version: sd_args.maybe_version.unwrap_or(0),
+    mysql_pool,
+  }).await?;
+
+  Ok(JobSuccessResult {
+    maybe_result_entity: Some(ResultEntity {
+      entity_type: InferenceResultType::UploadModel,
+      entity_token: model_weight_token_result.to_string(),
+    }),
+    inference_duration: Duration::from_secs(0),
+  })
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
salt	07d8125de9	remove excess message	2024-02-20 07:42:27 -08:00
salt	aa9e1c2fe4	new dependencies + download workflow from weight token	2024-02-20 07:41:05 -08:00
salt	48b1f6306b	save job args to file modification info	2024-02-17 06:21:51 -08:00
Brandon Thomas	d4bda54911	Merge pull request #301 from storytold/feb-14-mithoo bvh result type is video; mime type/extension is set correctly	2024-02-14 19:12:18 -05:00
Brandon Thomas	30cef2bc57	validate compositor http request	2024-02-14 18:52:45 -05:00
Brandon Thomas	e462f0c08a	Store download URL in top-level job info	2024-02-14 18:24:58 -05:00
Brandon Thomas	04c0e09d6f	styletts include text transcript	2024-02-14 17:41:06 -05:00
Brandon Thomas	319a8f0ba8	add text transcripts to tts inference jobs	2024-02-14 17:34:04 -05:00
Brandon Thomas	4fa4a0a4e1	post-migration widening of file_size_bytes	2024-02-14 11:06:31 -05:00
Kasisnu	89099a4ef8	bvh result type is video; mime type/extension is set correctly	2024-02-14 15:57:32 +05:30
Brandon Thomas	53b2aedbb0	Five new schema migrations	2024-02-14 05:27:01 -05:00
Brandon Thomas	00abb6230b	inference status - get model_weights title	2024-02-13 20:48:15 -05:00
Brandon Thomas	fa21d8251e	document job status endpoints	2024-02-13 20:31:59 -05:00
Brandon Thomas	17b38a62cf	add more staff accounts	2024-02-13 19:36:38 -05:00
Brandon Thomas	132230bbbe	enums got mixed up. we should rename these	2024-02-13 15:30:53 -05:00
Kasisnu	1b97d638a8	Merge pull request #300 from storytold/feb-13-mithoo return errors from conversion endpoints when media file isnt sent	2024-02-13 21:57:13 +05:30
Kasisnu	bd653c70d8	return errors from conversion endpoints when media file isnt sent	2024-02-13 21:56:36 +05:30
Brandon Thomas	d4a4ac4077	more users for the rollout	2024-02-12 12:34:51 -05:00
Brandon Thomas	32acf33eeb	after sd job, return the first media file token	2024-02-12 05:19:14 -05:00
Brandon Thomas	c9060d973f	feature flag more endpoints; don't return certain results in user list	2024-02-12 04:38:56 -05:00
Brandon Thomas	a64990230e	cleanup sd enqueue endpoint, remove flag check	2024-02-12 04:23:49 -05:00
Brandon Thomas	a3c6cc5b03	cache SD model weights in a cache directory	2024-02-11 07:19:41 -05:00
Brandon Thomas	f16ca22110	log downloads	2024-02-10 17:10:33 -05:00
Brandon Thomas	5fa8bb93ce	sensible default development configs (wip)	2024-02-10 16:53:08 -05:00
Brandon Thomas	91e8a51216	setup SD to use a predefined SD weight from our bucket	2024-02-10 16:47:21 -05:00
Brandon Thomas	7bd95d2078	split up sd job into three functions	2024-02-10 16:39:56 -05:00
Brandon Thomas	e0f141fa74	enable michael's account	2024-02-10 16:14:45 -05:00
Brandon Thomas	029798e665	codegen db - field widened?	2024-02-09 22:48:36 -05:00
Brandon Thomas	850e1b3c10	add UserDetailsLight + avatar to /v1/session endpoint + schema docs	2024-02-09 22:43:28 -05:00
Brandon Thomas	e1ea2cdf67	format api annotations	2024-02-09 22:02:43 -05:00
Brandon Thomas	74ed7f365b	UserDetailsLight refactor - consolidate, fix bug, get ready for reuse	2024-02-09 21:44:13 -05:00
Brandon Thomas	e327a8648c	simplify cli tool args	2024-02-09 01:04:15 -05:00
Kasisnu	88c68ccf9b	Merge pull request #293 from storytold/jan-31-mithoo scaffold for bvh -> workflow job	2024-02-09 00:06:05 +05:30
Kasisnu	2e5fdd91d3	scaffold for bvh -> workflow job	2024-02-08 23:46:22 +05:30
Brandon Thomas	44135d8d10	enroll users in studio	2024-02-07 22:27:03 -05:00
Brandon Thomas	0d7afb10c2	codegen queries	2024-02-07 00:42:48 -05:00
Brandon Thomas	dd76a6fea1	flag to session endpoint	2024-02-06 23:41:26 -05:00
Brandon Thomas	1886f40a9b	handle flag check for dynamic user session types	2024-02-06 23:30:05 -05:00
Brandon Thomas	f54ddb0acb	feature flag off certain endpoints	2024-02-06 23:20:14 -05:00
Brandon Thomas	a7b5cc9fc3	add is_deleted flag to es	2024-02-06 23:20:14 -05:00
Brandon Thomas	0e24e38576	rollout column for studio access	2024-02-06 23:20:14 -05:00
Kasisnu	6bc6bf0c24	Merge pull request #271 from storytold/otel-instrumentation Otel instrumentation - first pass	2024-02-06 09:58:59 +05:30
ArEnSc	fb2b5c9735	Merge pull request #297 from storytold/feature/downloader_fix made extension and suffix changes.	2024-02-05 21:13:23 -05:00
ArEnSc	3d04302de1	Ok tested seems like the data is added correctly.	2024-02-05 21:09:05 -05:00
ArEnSc	7e169e3b51	Testing	2024-02-05 20:45:07 -05:00
ArEnSc	5ba8d876f0	made extension and suggix changes.	2024-02-05 18:51:09 -05:00
Kasisnu	44845ad54e	otel: some stats we were already logging to stdout	2024-02-05 20:59:40 +05:30
Madhukar Mishra	6ca7cbee31	remove job.inference_job_token as a label for a metric	2024-02-05 18:18:16 +05:30
Madhukar Mishra	2cf821d450	fix - add scheme and port to otel collector endpoint	2024-02-05 18:18:16 +05:30
Madhukar Mishra	d5ef9c0b78	use a sync instrument - histogram	2024-02-05 18:18:16 +05:30
Madhukar Mishra	6284b2a32f	update Cargo.lock	2024-02-05 18:18:16 +05:30
Madhukar Mishra	a104ae37aa	init metrics provider	2024-02-05 18:18:16 +05:30
Madhukar Mishra	fa3f506f27	fixup	2024-02-05 18:18:16 +05:30
Madhukar Mishra	eb4f06179e	fixup	2024-02-05 18:18:16 +05:30
Madhukar Mishra	6c86764798	fixup	2024-02-05 18:18:16 +05:30
Madhukar Mishra	cfa9558a33	add a metric to try otel	2024-02-05 18:18:14 +05:30
Madhukar Mishra	2cadfb105b	update .gitignore	2024-02-05 18:17:31 +05:30
Madhukar Mishra	97560cb1f7	add otel crate	2024-02-05 18:17:28 +05:30
Brandon Thomas	7450dd1525	whoops	2024-02-05 05:00:37 -05:00
Brandon Thomas	d8e066dc23	codegen db	2024-02-05 04:50:46 -05:00
Brandon Thomas	fdae854e3f	build es-update-job in docker	2024-02-05 04:48:51 -05:00
Brandon Thomas	541695f028	Merge pull request #296 from storytold/brandon-es-job elasticsearch update job	2024-02-05 04:40:36 -05:00
Brandon Thomas	0719a929f8	pause on no updates	2024-02-05 04:37:04 -05:00
Brandon Thomas	c0c86c90b8	dev secrets	2024-02-05 04:11:49 -05:00
Brandon Thomas	a5c8f5fdc6	cleanup	2024-02-05 04:06:59 -05:00
Brandon Thomas	58210f0c25	job almost done	2024-02-05 04:00:37 -05:00
Brandon Thomas	ccf910e456	wip elasticsearch job	2024-02-04 21:57:27 -05:00
Brandon Thomas	d9197d2fd9	codegen db	2024-02-04 21:57:10 -05:00
Brandon Thomas	61d32a1790	more idempotency on more endpoints	2024-02-04 21:54:02 -05:00
Brandon Thomas	361b0e0894	generic idempotency token query; add to media file uploads	2024-02-04 21:45:21 -05:00
Brandon Thomas	12070bc399	indices on temporal columns in model_weights	2024-02-04 21:26:15 -05:00
Brandon Thomas	a49b8b4b74	logging	2024-02-04 02:58:03 -05:00
Brandon Thomas	2422f6d0aa	input.mp4 hardcoded	2024-02-04 02:21:52 -05:00
Brandon Thomas	a1fc56ecee	prompt argument for comfy	2024-02-04 01:48:05 -05:00
Brandon Thomas	05025d8817	maybe fix pathing	2024-02-04 01:04:43 -05:00
Brandon Thomas	bcea80c39b	get comfyui media downloads working	2024-02-04 00:43:46 -05:00
Brandon Thomas	cf59c8e2fc	extra logging	2024-02-03 23:49:51 -05:00
Brandon Thomas	5528993768	config for comfyui workflow	2024-02-03 23:44:21 -05:00
Brandon Thomas	da9e03e3ae	api for delete	2024-02-03 02:07:44 -05:00
Brandon Thomas	76b8b07d8a	6 sd batch size	2024-02-03 01:25:16 -05:00
Brandon Thomas	14cba8b8ed	fix sd inference pathing	2024-02-03 01:15:03 -05:00
Brandon Thomas	55805c2d75	safetensors -> pt x2	2024-02-03 00:42:16 -05:00
ArEnSc	ae80de8508	Merge pull request #295 from storytold/feature/comfyui support docker container & more comfyui dependencies	2024-02-03 00:37:05 -05:00
Brandon Thomas	b965bfb602	file_size_bytes needs to be BIGINT	2024-02-03 00:20:08 -05:00
				`@@ -0,0 +1 @@`
				`pub mod batch_get_model_weights_for_elastic_search_backfill;`
				`@@ -0,0 +1,2 @@`
				`# es-update-job default .env configurations (these can be overridden with k8s)`