Feedback addressed!

Added documentation about the documentation
documentation system open api spec.
2026-10-24 02:29:32 -04:00 · 2023-12-05 20:26:07 -05:00 · 2023-12-05 19:14:02 -05:00 · 2023-12-05 19:00:44 -05:00 · 2023-12-04 05:00:46 -05:00 · 2023-12-04 03:54:05 -05:00
129 changed files with 3246 additions and 736 deletions
@@ -1,12 +1,12 @@
 {
  "db_name": "MySQL",
-  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  inference_category = ?,\n  maybe_model_type = ?,\n  maybe_model_token = ?,\n\n  maybe_input_source_token = ?,\n  maybe_input_source_token_type = ?,\n\n  maybe_raw_inference_text = ?,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  priority_level = ?,\n  is_keepalive_required = ?,\n\n  is_debug_request = ?,\n  maybe_routing_tag = ?,\n\n  status = \"pending\"\n        ",
+  "query": "\nINSERT INTO generic_inference_jobs\nSET\n  token = ?,\n  uuid_idempotency_token = ?,\n\n  inference_category = ?,\n  maybe_model_type = ?,\n  maybe_model_token = ?,\n\n  maybe_input_source_token = ?,\n  maybe_input_source_token_type = ?,\n\n  maybe_raw_inference_text = ?,\n\n  maybe_inference_args = ?,\n\n  maybe_creator_user_token = ?,\n  maybe_creator_anonymous_visitor_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?,\n\n  priority_level = ?,\n  is_keepalive_required = ?,\n  max_duration_seconds = ?,\n\n  is_debug_request = ?,\n  maybe_routing_tag = ?,\n\n  status = \"pending\"\n        ",
  "describe": {
    "columns": [],
    "parameters": {
-      "Right": 17
+      "Right": 18
    },
    "nullable": []
  },
-  "hash": "b1750559783bcd55b2ae139372a4782ae363787a840a32f5c17c8408dbf15bc7"
+  "hash": "0d780caac7560f84efb50bd2ed876cbdd728b32e26b73dfbb3009ada12985405"
 }
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\n        INSERT INTO model_weights\n        SET\n          token = ?,\n          weights_type = ?,\n          weights_category = ?,\n          title = ?,\n          maybe_thumbnail_token = ?,\n          description_markdown = ?,\n          description_rendered_html = ?,\n          creator_user_token = ?,\n          creator_ip_address = ?,\n          creator_set_visibility = ?,\n          maybe_last_update_user_token = ?,\n          original_download_url = ?,\n          original_filename = ?,\n          file_size_bytes = ?,\n          file_checksum_sha2 = ?,\n          private_bucket_hash = ?,\n          maybe_private_bucket_prefix = ?,\n          maybe_private_bucket_extension = ?,\n          cached_user_ratings_total_count = ?,\n          cached_user_ratings_positive_count = ?,\n          cached_user_ratings_negative_count = ?,\n          maybe_cached_user_ratings_ratio = ?,\n          version = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 23
+    },
+    "nullable": []
+  },
+  "hash": "274cfb1e3caca706e3dbc3d95064e0f4b354c936f8de948070854920f265cc41"
+}
@@ -0,0 +1,355 @@
+{
+  "db_name": "MySQL",
+  "query": "\n            SELECT              \n                mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n                mw.title,\n                mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n                mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n                mw.maybe_thumbnail_token,\n                mw.description_markdown,\n                mw.description_rendered_html,\n                users.token as `creator_user_token: tokens::tokens::users::UserToken`,\n                users.username as creator_username,\n                users.display_name as creator_display_name,\n                users.email_gravatar_hash as creator_email_gravatar_hash,\n                mw.creator_ip_address,\n                mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n                mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n                mw.original_download_url,\n                mw.original_filename,\n                mw.file_size_bytes,\n                mw.file_checksum_sha2,\n                mw.private_bucket_hash,\n                mw.maybe_private_bucket_prefix,\n                mw.maybe_private_bucket_extension,\n                mw.cached_user_ratings_negative_count,\n                mw.cached_user_ratings_positive_count,\n                mw.cached_user_ratings_total_count,\n                mw.maybe_cached_user_ratings_ratio,\n                mw.cached_user_ratings_last_updated_at,\n                mw.version,\n                mw.created_at,\n                mw.updated_at,\n                mw.user_deleted_at,\n                mw.mod_deleted_at\n            FROM model_weights as mw\n            JOIN users\n                ON users.token = mw.creator_user_token\n            WHERE\n                users.username = ?\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::model_weights::ModelWeightToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "title",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "weights_type: enums::by_table::model_weights::weights_types::WeightsType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "weights_category: enums::by_table::model_weights::weights_category::WeightsCategory",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_thumbnail_token",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "description_markdown",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "description_rendered_html",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 8,
+        "name": "creator_username",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 80
+        }
+      },
+      {
+        "ordinal": 9,
+        "name": "creator_display_name",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 80
+        }
+      },
+      {
+        "ordinal": 10,
+        "name": "creator_email_gravatar_hash",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 11,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 12,
+        "name": "creator_set_visibility: enums::common::visibility::Visibility",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | ENUM",
+          "char_set": 224,
+          "max_size": 28
+        }
+      },
+      {
+        "ordinal": 13,
+        "name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 14,
+        "name": "original_download_url",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 2048
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "original_filename",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 16,
+        "name": "file_size_bytes",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 17,
+        "name": "file_checksum_sha2",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 256
+        }
+      },
+      {
+        "ordinal": 18,
+        "name": "private_bucket_hash",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "maybe_private_bucket_prefix",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 20,
+        "name": "maybe_private_bucket_extension",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 21,
+        "name": "cached_user_ratings_negative_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 22,
+        "name": "cached_user_ratings_positive_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 23,
+        "name": "cached_user_ratings_total_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 24,
+        "name": "maybe_cached_user_ratings_ratio",
+        "type_info": {
+          "type": "Float",
+          "flags": "",
+          "char_set": 63,
+          "max_size": 12
+        }
+      },
+      {
+        "ordinal": 25,
+        "name": "cached_user_ratings_last_updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 26,
+        "name": "version",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 11
+        }
+      },
+      {
+        "ordinal": 27,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 28,
+        "name": "updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 29,
+        "name": "user_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 30,
+        "name": "mod_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "3d586592422727dcc928574966bdf5081ac69fdbb856fffcbd68187fd99c4642"
+}
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE model_weights\nSET\n  mod_deleted_at = NULL\nWHERE\n  token = ?\nLIMIT 1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": []
+  },
+  "hash": "3dfc7f22c16543116c4ca17f5bee0336f5cfa1522681ba1bb6b1874cd8f13297"
+}
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\n        UPDATE model_weights\n        SET\n            title = ?,\n            description_markdown = ?,\n            maybe_thumbnail_token = ?,\n            description_rendered_html = ?,\n            creator_set_visibility = ?,\n            version = version + 1\n        WHERE token = ?\n        LIMIT 1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 6
+    },
+    "nullable": []
+  },
+  "hash": "4ed42766cc0ca2234b7bc1837d645d05198d483564b39a2a8b8034df25e287a4"
+}
@@ -0,0 +1,322 @@
+{
+  "db_name": "MySQL",
+  "query": "\n        SELECT\n        wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n        wt.title,\n        wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n        wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n        wt.maybe_thumbnail_token,\n        wt.description_markdown,\n        wt.description_rendered_html,\n        wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n        wt.creator_ip_address,\n        wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n        wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n        wt.original_download_url,\n        wt.original_filename,\n        wt.file_size_bytes,\n        wt.file_checksum_sha2,\n        wt.private_bucket_hash,\n        wt.maybe_private_bucket_prefix,\n        wt.maybe_private_bucket_extension,\n        wt.cached_user_ratings_negative_count,\n        wt.cached_user_ratings_positive_count,\n        wt.cached_user_ratings_total_count,\n        wt.maybe_cached_user_ratings_ratio,\n        wt.cached_user_ratings_last_updated_at,\n        wt.version,\n        wt.created_at,\n        wt.updated_at,\n        wt.user_deleted_at,\n        wt.mod_deleted_at\n        FROM model_weights as wt\n        WHERE\n            wt.token = ?\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::model_weights::ModelWeightToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "title",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "weights_type: enums::by_table::model_weights::weights_types::WeightsType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "weights_category: enums::by_table::model_weights::weights_category::WeightsCategory",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_thumbnail_token",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "description_markdown",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "description_rendered_html",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 8,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 9,
+        "name": "creator_set_visibility: enums::common::visibility::Visibility",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | ENUM",
+          "char_set": 224,
+          "max_size": 28
+        }
+      },
+      {
+        "ordinal": 10,
+        "name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 11,
+        "name": "original_download_url",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 2048
+        }
+      },
+      {
+        "ordinal": 12,
+        "name": "original_filename",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 13,
+        "name": "file_size_bytes",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 14,
+        "name": "file_checksum_sha2",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 256
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "private_bucket_hash",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 16,
+        "name": "maybe_private_bucket_prefix",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 17,
+        "name": "maybe_private_bucket_extension",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 18,
+        "name": "cached_user_ratings_negative_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "cached_user_ratings_positive_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 20,
+        "name": "cached_user_ratings_total_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 21,
+        "name": "maybe_cached_user_ratings_ratio",
+        "type_info": {
+          "type": "Float",
+          "flags": "",
+          "char_set": 63,
+          "max_size": 12
+        }
+      },
+      {
+        "ordinal": 22,
+        "name": "cached_user_ratings_last_updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 23,
+        "name": "version",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 11
+        }
+      },
+      {
+        "ordinal": 24,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 25,
+        "name": "updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 26,
+        "name": "user_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 27,
+        "name": "mod_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "6a94791c8a8f66ee54bb367a4e8c71106bbcbbb205f54c14e3657279e83dabb6"
+}
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE model_weights\nSET\n  user_deleted_at = CURRENT_TIMESTAMP\nWHERE\n  token = ?\nLIMIT 1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": []
+  },
+  "hash": "7487c997706788c670a76da81ce90532b9748c8afcc707436d1f04ed976ce30e"
+}
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE model_weights\nSET\n  user_deleted_at = NULL\nWHERE\n  token = ?\nLIMIT 1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": []
+  },
+  "hash": "7dbfbb21d502aad7859016fd287c57010296011928cf21b036a8dc57f1ef619d"
+}
@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nUPDATE model_weights\nSET\n  mod_deleted_at = CURRENT_TIMESTAMP\nWHERE\n  token = ?\nLIMIT 1\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": []
+  },
+  "hash": "9d2c8d49cf9840a547230e5c6febc041f39cd069bcb7a4a94ffb4f1cb3be6d7c"
+}
@@ -0,0 +1,355 @@
+{
+  "db_name": "MySQL",
+  "query": "\n            SELECT\n                mw.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n                mw.title,\n                mw.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n                mw.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n                mw.maybe_thumbnail_token,\n                mw.description_markdown,\n                mw.description_rendered_html,\n                users.token as `creator_user_token: tokens::tokens::users::UserToken`,\n                users.username as creator_username,\n                users.display_name as creator_display_name,\n                users.email_gravatar_hash as creator_email_gravatar_hash,\n                mw.creator_ip_address,\n                mw.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n                mw.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n                mw.original_download_url,\n                mw.original_filename,\n                mw.file_size_bytes,\n                mw.file_checksum_sha2,\n                mw.private_bucket_hash,\n                mw.maybe_private_bucket_prefix,\n                mw.maybe_private_bucket_extension,\n                mw.cached_user_ratings_negative_count,\n                mw.cached_user_ratings_positive_count,\n                mw.cached_user_ratings_total_count,\n                mw.maybe_cached_user_ratings_ratio,\n                mw.cached_user_ratings_last_updated_at,\n                mw.version,\n                mw.created_at,\n                mw.updated_at,\n                mw.user_deleted_at,\n                mw.mod_deleted_at\n            FROM model_weights as mw\n            JOIN users\n                ON users.token = mw.creator_user_token\n            WHERE\n                users.username = ?\n                AND mw.user_deleted_at IS NULL\n                AND mw.mod_deleted_at IS NULL\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::model_weights::ModelWeightToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "title",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "weights_type: enums::by_table::model_weights::weights_types::WeightsType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "weights_category: enums::by_table::model_weights::weights_category::WeightsCategory",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_thumbnail_token",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "description_markdown",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "description_rendered_html",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 8,
+        "name": "creator_username",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 80
+        }
+      },
+      {
+        "ordinal": 9,
+        "name": "creator_display_name",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 80
+        }
+      },
+      {
+        "ordinal": 10,
+        "name": "creator_email_gravatar_hash",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 11,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 12,
+        "name": "creator_set_visibility: enums::common::visibility::Visibility",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | ENUM",
+          "char_set": 224,
+          "max_size": 28
+        }
+      },
+      {
+        "ordinal": 13,
+        "name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 14,
+        "name": "original_download_url",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 2048
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "original_filename",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 16,
+        "name": "file_size_bytes",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 17,
+        "name": "file_checksum_sha2",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 256
+        }
+      },
+      {
+        "ordinal": 18,
+        "name": "private_bucket_hash",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "maybe_private_bucket_prefix",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 20,
+        "name": "maybe_private_bucket_extension",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 21,
+        "name": "cached_user_ratings_negative_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 22,
+        "name": "cached_user_ratings_positive_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 23,
+        "name": "cached_user_ratings_total_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 24,
+        "name": "maybe_cached_user_ratings_ratio",
+        "type_info": {
+          "type": "Float",
+          "flags": "",
+          "char_set": 63,
+          "max_size": 12
+        }
+      },
+      {
+        "ordinal": 25,
+        "name": "cached_user_ratings_last_updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 26,
+        "name": "version",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 11
+        }
+      },
+      {
+        "ordinal": 27,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 28,
+        "name": "updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 29,
+        "name": "user_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 30,
+        "name": "mod_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "a6ed8524fb5339fed17bec6611f8f56b6bdee35bac63c9eacbe21eb032a6a831"
+}
@@ -0,0 +1,322 @@
+{
+  "db_name": "MySQL",
+  "query": "\n        SELECT\n        wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,\n        wt.title,\n        wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,\n        wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,\n        wt.maybe_thumbnail_token,\n        wt.description_markdown,\n        wt.description_rendered_html,\n        wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,\n        wt.creator_ip_address,\n        wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,\n        wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,\n        wt.original_download_url,\n        wt.original_filename,\n        wt.file_size_bytes,\n        wt.file_checksum_sha2,\n        wt.private_bucket_hash,\n        wt.maybe_private_bucket_prefix,\n        wt.maybe_private_bucket_extension,\n        wt.cached_user_ratings_negative_count,\n        wt.cached_user_ratings_positive_count,\n        wt.cached_user_ratings_total_count,\n        wt.maybe_cached_user_ratings_ratio,\n        wt.cached_user_ratings_last_updated_at,\n        wt.version,\n        wt.created_at,\n        wt.updated_at,\n        wt.user_deleted_at,\n        wt.mod_deleted_at\n        FROM model_weights as wt\n        WHERE\n            wt.token = ?\n            AND wt.user_deleted_at IS NULL\n            AND wt.mod_deleted_at IS NULL\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "token: tokens::tokens::model_weights::ModelWeightToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "title",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 2,
+        "name": "weights_type: enums::by_table::model_weights::weights_types::WeightsType",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 3,
+        "name": "weights_category: enums::by_table::model_weights::weights_category::WeightsCategory",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "maybe_thumbnail_token",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 5,
+        "name": "description_markdown",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 6,
+        "name": "description_rendered_html",
+        "type_info": {
+          "type": "Blob",
+          "flags": "NOT_NULL | BLOB | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 262140
+        }
+      },
+      {
+        "ordinal": 7,
+        "name": "creator_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 8,
+        "name": "creator_ip_address",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 160
+        }
+      },
+      {
+        "ordinal": 9,
+        "name": "creator_set_visibility: enums::common::visibility::Visibility",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | ENUM",
+          "char_set": 224,
+          "max_size": 28
+        }
+      },
+      {
+        "ordinal": 10,
+        "name": "maybe_last_update_user_token: tokens::tokens::users::UserToken",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 11,
+        "name": "original_download_url",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 2048
+        }
+      },
+      {
+        "ordinal": 12,
+        "name": "original_filename",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 1020
+        }
+      },
+      {
+        "ordinal": 13,
+        "name": "file_size_bytes",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 14,
+        "name": "file_checksum_sha2",
+        "type_info": {
+          "type": "String",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 256
+        }
+      },
+      {
+        "ordinal": 15,
+        "name": "private_bucket_hash",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | BINARY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 128
+        }
+      },
+      {
+        "ordinal": 16,
+        "name": "maybe_private_bucket_prefix",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 17,
+        "name": "maybe_private_bucket_extension",
+        "type_info": {
+          "type": "VarString",
+          "flags": "BINARY",
+          "char_set": 224,
+          "max_size": 64
+        }
+      },
+      {
+        "ordinal": 18,
+        "name": "cached_user_ratings_negative_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 19,
+        "name": "cached_user_ratings_positive_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 20,
+        "name": "cached_user_ratings_total_count",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | UNSIGNED",
+          "char_set": 63,
+          "max_size": 10
+        }
+      },
+      {
+        "ordinal": 21,
+        "name": "maybe_cached_user_ratings_ratio",
+        "type_info": {
+          "type": "Float",
+          "flags": "",
+          "char_set": 63,
+          "max_size": 12
+        }
+      },
+      {
+        "ordinal": 22,
+        "name": "cached_user_ratings_last_updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 23,
+        "name": "version",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 11
+        }
+      },
+      {
+        "ordinal": 24,
+        "name": "created_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 25,
+        "name": "updated_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "NOT_NULL | BINARY | TIMESTAMP | ON_UPDATE_NOW",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 26,
+        "name": "user_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      },
+      {
+        "ordinal": 27,
+        "name": "mod_deleted_at",
+        "type_info": {
+          "type": "Timestamp",
+          "flags": "BINARY",
+          "char_set": 63,
+          "max_size": 19
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      false,
+      false,
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "f0b5767d52c389660d1859e628c746f6f33a9e1c25f21f66aeeb0784df768c8e"
+}
@@ -736,12 +736,6 @@ dependencies = [
 "rustc-demangle",
 ]

-[[package]]
-name = "base-x"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
-
 [[package]]
 name = "base64"
 version = "0.11.0"
@@ -1175,6 +1169,15 @@ dependencies = [
 "bitflags 1.3.2",
 ]

+[[package]]
+name = "collections"
+version = "0.0.1"
+dependencies = [
+ "errors",
+ "my-workspace-hack",
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "color_quant"
 version = "1.1.0"
@@ -1253,12 +1256,6 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f"

-[[package]]
-name = "const_fn"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935"
-
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -1867,12 +1864,6 @@ dependencies = [
 "winapi 0.3.9",
 ]

-[[package]]
-name = "discard"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
-
 [[package]]
 name = "displaydoc"
 version = "0.2.4"
@@ -2003,7 +1994,7 @@ name = "easyenv"
 version = "0.1.0"
 dependencies = [
 "dotenv",
- "env_logger 0.10.1",
+ "env_logger",
 "log",
 "my-workspace-hack",
 "storyteller_root",
@@ -2184,26 +2175,13 @@ dependencies = [
 "toml",
 ]

-[[package]]
-name = "env_logger"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
-dependencies = [
- "atty",
- "humantime 1.3.0",
- "log",
- "regex",
- "termcolor",
-]
-
 [[package]]
 name = "env_logger"
 version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"
 dependencies = [
- "humantime 2.1.0",
+ "humantime",
 "is-terminal",
 "log",
 "regex",
@@ -2296,6 +2274,7 @@ name = "filesys"
 version = "0.0.1"
 dependencies = [
 "errors",
+ "log",
 "my-workspace-hack",
 ]

@@ -2848,15 +2827,6 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"

-[[package]]
-name = "humantime"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
-dependencies = [
- "quick-error",
-]
-
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -2994,6 +2964,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
 dependencies = [
 "equivalent",
 "hashbrown 0.14.2",
+ "serde",
 ]

 [[package]]
@@ -3032,9 +3003,9 @@ dependencies = [
 "buckets",
 "chrono",
 "cloud_storage",
+ "collections",
 "concurrency",
 "config",
- "container_common",
 "dotenv",
 "easyenv",
 "enums",
@@ -3045,6 +3016,7 @@ dependencies = [
 "hyper",
 "images",
 "jobs_common",
+ "lfu",
 "log",
 "media",
 "memory_caching",
@@ -3399,7 +3371,7 @@ name = "logging"
 version = "0.0.1"
 dependencies = [
 "dotenv",
- "env_logger 0.10.1",
+ "env_logger",
 "log",
 "my-workspace-hack",
 ]
@@ -3755,7 +3727,7 @@ dependencies = [
 "syn 1.0.109",
 "syn 2.0.39",
 "time 0.3.30",
- "time-macros 0.2.15",
+ "time-macros",
 "tokio",
 "tokio-stream",
 "tracing",
@@ -3792,7 +3764,7 @@ dependencies = [
 "serial_test",
 "sqlx",
 "tokens",
- "tokio 1.34.0",
+ "tokio",
 "toml",
 ]

@@ -3852,14 +3824,14 @@ name = "newrelic-telemetry"
 version = "0.1.0"
 dependencies = [
 "anyhow",
- "env_logger 0.7.1",
+ "env_logger",
 "flate2",
 "futures 0.3.29",
 "hyper",
 "hyper-tls",
 "log",
 "my-workspace-hack",
- "rand 0.7.3",
+ "rand 0.8.5",
 "serde",
 "serde_json",
 "tokio",
@@ -4423,10 +4395,28 @@ dependencies = [
 ]

 [[package]]
-name = "proc-macro-hack"
-version = "0.5.20+deprecated"
+name = "proc-macro-error"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]

 [[package]]
 name = "proc-macro2"
@@ -4467,12 +4457,6 @@ dependencies = [
 "bytemuck",
 ]

-[[package]]
-name = "quick-error"
-version = "1.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
-
 [[package]]
 name = "quick-xml"
 version = "0.26.0"
@@ -4736,7 +4720,7 @@ dependencies = [
 "serde",
 "serde_derive",
 "serde_json",
- "time 0.2.27",
+ "time 0.3.30",
 ]

 [[package]]
@@ -4917,6 +4901,41 @@ dependencies = [
 "quick-xml 0.30.0",
 ]

+[[package]]
+name = "rust-embed"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1e7d90385b59f0a6bf3d3b757f3ca4ece2048265d70db20a2016043d4509a40"
+dependencies = [
+ "rust-embed-impl",
+ "rust-embed-utils",
+ "walkdir",
+]
+
+[[package]]
+name = "rust-embed-impl"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rust-embed-utils",
+ "shellexpand",
+ "syn 2.0.39",
+ "walkdir",
+]
+
+[[package]]
+name = "rust-embed-utils"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "873feff8cb7bf86fdf0a71bb21c95159f4e4a37dd7a4bd1855a940909b583ada"
+dependencies = [
+ "sha2 0.10.8",
+ "walkdir",
+]
+
 [[package]]
 name = "rust-ini"
 version = "0.18.0"
@@ -5452,6 +5471,15 @@ dependencies = [
 "digest 0.10.7",
 ]

+[[package]]
+name = "shellexpand"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4"
+dependencies = [
+ "dirs",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.1"
@@ -5836,70 +5864,12 @@ dependencies = [
 "winapi 0.3.9",
 ]

-[[package]]
-name = "standback"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff"
-dependencies = [
- "version_check",
-]
-
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"

-[[package]]
-name = "stdweb"
-version = "0.4.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
-dependencies = [
- "discard",
- "rustc_version 0.2.3",
- "stdweb-derive",
- "stdweb-internal-macros",
- "stdweb-internal-runtime",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "stdweb-derive"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
-dependencies = [
- "proc-macro2",
- "quote",
- "serde",
- "serde_derive",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "stdweb-internal-macros"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
-dependencies = [
- "base-x",
- "proc-macro2",
- "quote",
- "serde",
- "serde_derive",
- "serde_json",
- "sha1 0.6.1",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "stdweb-internal-runtime"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
-
 [[package]]
 name = "storyteller-web"
 version = "0.0.1"
@@ -5983,7 +5953,7 @@ dependencies = [
 "speculoos",
 "sqlx",
 "symphonia",
- "time 0.2.27",
+ "time 0.3.30",
 "tokens",
 "tokio",
 "toml",
@@ -5993,6 +5963,8 @@ dependencies = [
 "url_config",
 "user_input_common",
 "users_component",
+ "utoipa",
+ "utoipa-swagger-ui",
 "uuid 0.8.2",
 ]

@@ -6429,21 +6401,6 @@ dependencies = [
 "winapi 0.3.9",
 ]

-[[package]]
-name = "time"
-version = "0.2.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242"
-dependencies = [
- "const_fn",
- "libc",
- "standback",
- "stdweb",
- "time-macros 0.1.1",
- "version_check",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "time"
 version = "0.3.30"
@@ -6455,7 +6412,7 @@ dependencies = [
 "powerfmt",
 "serde",
 "time-core",
- "time-macros 0.2.15",
+ "time-macros",
 ]

 [[package]]
@@ -6464,16 +6421,6 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"

-[[package]]
-name = "time-macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1"
-dependencies = [
- "proc-macro-hack",
- "time-macros-impl",
-]
-
 [[package]]
 name = "time-macros"
 version = "0.2.15"
@@ -6483,19 +6430,6 @@ dependencies = [
 "time-core",
 ]

-[[package]]
-name = "time-macros-impl"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f"
-dependencies = [
- "proc-macro-hack",
- "proc-macro2",
- "quote",
- "standback",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
@@ -6524,6 +6458,7 @@ dependencies = [
 "sqlx-core",
 "strum",
 "toml",
+ "utoipa",
 ]

 [[package]]
@@ -6777,6 +6712,7 @@ dependencies = [
 "dotenv",
 "easyenv",
 "enums",
+ "filesys",
 "google_drive_common",
 "hashing",
 "hostname",
@@ -6803,10 +6739,12 @@ dependencies = [
 "chrono",
 "clap 2.34.0",
 "cloud_storage",
+ "collections",
 "config",
 "container_common",
 "dotenv",
 "easyenv",
+ "filesys",
 "hashing",
 "hostname",
 "hyper",
@@ -6874,6 +6812,7 @@ dependencies = [
 "anyhow",
 "base64 0.21.5",
 "chrono",
+ "collections",
 "config",
 "container_common",
 "diesel",
@@ -6902,7 +6841,7 @@ dependencies = [
 "serde_json",
 "sha2 0.9.9",
 "sqlx",
- "time 0.2.27",
+ "time 0.3.30",
 "tokens",
 "tokio",
 "tokio-tungstenite",
@@ -7140,7 +7079,7 @@ dependencies = [
 "sha2 0.9.9",
 "sqlx",
 "strum_macros",
- "time 0.2.27",
+ "time 0.3.30",
 "tokens",
 "toml",
 "user_input_common",
@@ -7158,6 +7097,47 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"

+[[package]]
+name = "utoipa"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_json",
+ "utoipa-gen",
+]
+
+[[package]]
+name = "utoipa-gen"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d"
+dependencies = [
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "syn 2.0.39",
+]
+
+[[package]]
+name = "utoipa-swagger-ui"
+version = "5.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51"
+dependencies = [
+ "actix-web",
+ "mime_guess",
+ "regex",
+ "rust-embed",
+ "serde",
+ "serde_json",
+ "utoipa",
+ "zip",
+]
+
 [[package]]
 name = "uuid"
 version = "0.8.2"
@@ -7209,6 +7189,7 @@ dependencies = [
 "container_common",
 "dotenv",
 "easyenv",
+ "filesys",
 "google_drive_common",
 "hashing",
 "hostname",
@@ -7238,6 +7219,7 @@ dependencies = [
 "container_common",
 "dotenv",
 "easyenv",
+ "filesys",
 "google_drive_common",
 "hostname",
 "hyper",
@@ -7480,7 +7462,7 @@ dependencies = [
 "serde_json",
 "sha2 0.9.9",
 "sqlx",
- "time 0.2.27",
+ "time 0.3.30",
 "tokio",
 "toml",
 "twitch_api2",
@@ -27,6 +27,7 @@ members = [
  "crates/lib/bootstrap",
  "crates/lib/caching/memory_caching",
  "crates/lib/caching/redis_caching",
+  "crates/lib/collections",
  "crates/lib/concurrency",
  "crates/lib/cookies",
  "crates/lib/crockford",
@@ -29,7 +29,9 @@ Web API Overview
        * `tts-inference-job` - (*deprecated*) Async TTS inference of Tacotron2 models only
        * `w2l-download-job` - (*deprecated*) Async download of videos and images
        * `w2l-inference-job` - (*deprecated*) Async W2L inference
-
+    * Documentation Server:
+        * `docs` - this will run a local swagger server with the endpoints that you have access to, http://127.0.0.1:8989/api-docs/openapi.json use this to port everything over to the post man.
+        
 ### PowerStream

 * [PowerStream](https://power.stream) (deprecated) is powered by the following applications:
@@ -380,3 +380,16 @@ of this error message. You can essentially ignore it.
 ### [Fix] Can't connect to local MySQL after install on Ubuntu

 - If MySql in local dev can't be connected to, [reset the accounts](https://linuxconfig.org/how-to-reset-root-mysql-mariadb-password-on-ubuntu-20-04-focal-fossa-linux).
+
+### API Server
+```
+cargo run --bin docs
+```
+This setups an api server with swagger docs to show you responses and requests with each associated endpoint
+It is run at http://127.0.0.1:8989/
+To obtain the json for the postman collection go to this url http://127.0.0.1:8989/api-docs/openapi.json
+We use this: `https://github.com/juhaku/utoipa`
+The documentation example is in: 
+`crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs`
+Write into the api doc to: 
+`crates/service/web/storyteller_web/src/api_doc.rs`
@@ -8,11 +8,25 @@ update generic_inference_jobs set status = 'dead' where maybe_routing_tag IS NOT
 -- Get pending jobs
 select count(*) from generic_inference_jobs where status = 'pending';

+-- Kill outstanding jobs
 update generic_inference_jobs
 set status = 'dead'
 where status IN ('pending', 'started', 'attempt_failed');

+-- Kill outstanding jobs of type
 update generic_inference_jobs
 set status = 'dead'
 where status IN ('pending', 'started', 'attempt_failed')
 and maybe_model_type IN ('sad_talker', 'so_vits_svc');
+
+-- Tacotron 2 TTS
+update generic_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed')
+  and maybe_model_type IN ('tacotron2');
+
+-- Sad Talker Face Animation
+update generic_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed')
+  and maybe_model_type IN ('sad_talker');
@@ -7,7 +7,7 @@ set -euxo pipefail

 # NB: Database used by desktop app for AiChatBot sidecar.
 # Make sure it has been migrated (use aichatbot-sidecar app directory as root).
-SQLITE_DATABASE_PATH=./runtime_data/database.db
+# SQLITE_DATABASE_PATH=./runtime_data/database.db

 # NB: OpenSSL is broken on Ubuntu 22.04 (this is documented elsewhere)
 # See https://askubuntu.com/a/1403961
@@ -133,7 +133,7 @@ mod tests {
    fn all_variants() {
      // Static check
      let mut variants = IdCategory::all_variants();
-      assert_eq!(variants.len(), 6);
+      assert_eq!(variants.len(), 7);
      assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile));
      assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult));
      assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult));
@@ -18,7 +18,7 @@ serde = "1.0.125"
 serde_derive = "1.0.125"

 # Random
-rand = "0.8.3"
+rand = "0.8.5"

 # MySql Database
 # "offline" supports typesafe builds without a live database
@@ -28,6 +28,8 @@ sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls",
 sqlx-core = { version = "0.7.2" }
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

+utoipa = { version = "4", features = ["actix_extras"] }
+
 [dev-dependencies]

 # Enum utilities
@@ -4,9 +4,10 @@ use serde::Deserialize;
 use serde::Serialize;

 use crate::prefixes::TokenPrefix;
+use utoipa::ToSchema;

 /// The primary key for "generic" inference jobs.
-#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize,ToSchema)]
 #[sqlx(transparent)]
 pub struct InferenceJobToken(String);

@@ -5,19 +5,14 @@ use sqlx::mysql::MySqlPoolOptions;
 use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
 use errors::AnyhowResult;

-
-use crate::seeding::model_weights::seed_weights;
-use crate::seeding::model_weights::seed_weights_for_user_token;
-
 use crate::bucket_clients::get_bucket_clients;
 use crate::cli_args::parse_cli_args;
+use crate::seeding::model_weights::seed_weights;
 use crate::seeding::tts_tacotron2::seed_tts_tacotron2;
-
 use crate::seeding::users::seed_user_accounts;
 use crate::seeding::voice_conversion::seed_voice_conversion;
 use crate::seeding::zero_shot_tts::seed_zero_shot_tts;

-
 pub mod bucket_clients;

 pub mod cli_args;
@@ -21,7 +21,7 @@ errors = { path = "../../lib/errors" }
 # External
 log = "0.4.20"
 once_cell = "1.18.0"
-rand = "0.8.1"
+rand = "0.8.5"
 regex = "1.10.0"
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

@@ -0,0 +1,26 @@
+[package]
+name = "collections"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+# NB: Public "collections" crate is not yet taken (!!), so we're fine to use this internally for now.
+name = "collections"
+path = "src/lib.rs"
+
+[dependencies]
+# Internal
+errors = { path = "../errors" }
+my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
+
+# External
+# KEEP THIS TO A MINIMUM OR NIL.
+rand = "0.8.5"
+
+[dev-dependencies]
+# None yet
@@ -0,0 +1,22 @@
+//! collections
+//!
+//! The purpose of this library is to provide "extra" data structures or utilities related to
+//! data structures. There should be minimal dependency introduction here.
+//!
+
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod multiple_random_from_vec;
+pub mod random_from_vec;
@@ -12,7 +12,7 @@ pub fn multiple_random_from_vec<T>(vec: &Vec<T>, amount: usize) -> Vec<&T> {

 #[cfg(test)]
 mod tests {
-  use crate::collections::multiple_random_from_vec::multiple_random_from_vec;
+  use crate::multiple_random_from_vec::multiple_random_from_vec;

  #[test]
  fn pulls_item_from_vec() {
@@ -7,7 +7,7 @@ pub fn random_from_vec<T>(vec: &Vec<T>) -> Option<&T> {

 #[cfg(test)]
 mod tests {
-  use crate::collections::random_from_vec::random_from_vec;
+  use crate::random_from_vec::random_from_vec;

  #[test]
  fn pulls_item_from_vec() {
@@ -15,7 +15,7 @@ name = "crockford"
 path = "src/lib.rs"

 [dependencies]
-rand = "0.8.3"
+rand = "0.8.5"
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

 [dev-dependencies]
@@ -17,7 +17,7 @@ path = "src/lib.rs"
 # External
 anyhow = "1.0.75"
 once_cell = "1.18.0"
-rand = "0.8.3"
+rand = "0.8.5"
 uuid = { version = "0.8.2", features = ["v4"] }
 log = "0.4.14"

@@ -1,2 +0,0 @@
-pub mod multiple_random_from_vec;
-pub mod random_from_vec;
@@ -1,15 +0,0 @@
-use std::path::PathBuf;
-
-use anyhow::bail;
-
-use crate::anyhow_result::AnyhowResult;
-
-pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> {
-  if !path.exists() {
-    bail!("Path doesn't exist: {:?}", path);
-  }
-  if !path.is_dir() {
-    bail!("Path isn't a directory: {:?}", path);
-  }
-  Ok(())
-}
@@ -1,5 +0,0 @@
-pub mod check_directory_exists;
-pub mod check_file_exists;
-pub mod safe_delete_possible_temp_file;
-pub mod safe_delete_temp_directory;
-pub mod safe_delete_temp_file;
@@ -1,7 +1,5 @@
 pub mod anyhow_result;
-pub mod collections;
 pub mod files;
-pub mod filesystem;
 pub mod i18n;
 pub mod numerics;
 pub mod thread;
@@ -1,3 +1,2 @@
 pub mod random_crockford_token;
-pub mod random_prefix_crockford_token;
 pub mod random_uuid;
@@ -1,100 +0,0 @@
-use anyhow::anyhow;
-use rand::Rng;
-
-use crate::anyhow_result::AnyhowResult;
-
-// Crockford characters
-const CROCKFORD_UPPERCASE_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ";
-
-const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
-
-// Random part can't be any less than this.
-const MIN_ENTROPY_LENGTH : usize = 8;
-
-#[deprecated(note = "see internal 'crockford' and 'token' crates")]
-pub fn random_prefix_crockford_token(prefix: &str, length: usize, uppercase: bool) -> AnyhowResult<String> {
-  let mut prefix = prefix.trim().to_string();
-
-  if prefix.is_empty() {
-    return Err(anyhow!("prefix not set; use other crockford functions"));
-  }
-
-  if !prefix.ends_with(":") {
-    prefix = format!("{}:", prefix);
-  }
-
-  let entropy_length : usize = length.saturating_sub(prefix.len());
-
-  if entropy_length < MIN_ENTROPY_LENGTH {
-    return Err(anyhow!("not enough entropy in token"));
-  }
-
-  let mut rng = rand::thread_rng();
-
-  let charset = if uppercase { CROCKFORD_UPPERCASE_CHARSET } else { CROCKFORD_LOWERCASE_CHARSET };
-
-  let entropy_part: String = (0..entropy_length)
-    .map(|_| {
-      let idx = rng.gen_range(0..charset.len());
-      charset[idx] as char
-    })
-    .collect();
-
-  let token = format!("{}{}", prefix, entropy_part);
-  Ok(token)
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::token::random_prefix_crockford_token::random_prefix_crockford_token;
-
-  #[test]
-  fn no_prefix_error() {
-    assert!(random_prefix_crockford_token("", 0, false).is_err());
-    assert!(random_prefix_crockford_token("", 10, false).is_err());
-    assert!(random_prefix_crockford_token("", 32, false).is_err());
-    assert!(random_prefix_crockford_token("", 0, true).is_err());
-    assert!(random_prefix_crockford_token("", 10, true).is_err());
-    assert!(random_prefix_crockford_token("", 32, true).is_err());
-  }
-
-  #[test]
-  fn not_enough_entropy_error() {
-    assert!(random_prefix_crockford_token("foo", 0, false).is_err());
-    assert!(random_prefix_crockford_token("foo", 5, false).is_err());
-    assert!(random_prefix_crockford_token("foo", 11, false).is_err());
-    assert!(random_prefix_crockford_token("foo:", 11, false).is_err());
-    assert!(random_prefix_crockford_token("foo", 0, true).is_err());
-    assert!(random_prefix_crockford_token("foo", 5, true).is_err());
-    assert!(random_prefix_crockford_token("foo", 11, true).is_err());
-    assert!(random_prefix_crockford_token("foo:", 11, true).is_err());
-  }
-
-  #[test]
-  fn starts_with() {
-    assert!(random_prefix_crockford_token("w", 10, false).unwrap().starts_with("w:"));
-    assert!(random_prefix_crockford_token("w", 10, false).unwrap().starts_with("w:"));
-    assert!(random_prefix_crockford_token("prefix", 16, false).unwrap().starts_with("prefix:"));
-    assert!(random_prefix_crockford_token("prefix:", 16, false).unwrap().starts_with("prefix:"));
-    assert!(random_prefix_crockford_token("w", 10, true).unwrap().starts_with("w:"));
-    assert!(random_prefix_crockford_token("w", 10, true).unwrap().starts_with("w:"));
-    assert!(random_prefix_crockford_token("prefix", 16, true).unwrap().starts_with("prefix:"));
-    assert!(random_prefix_crockford_token("prefix:", 16, true).unwrap().starts_with("prefix:"));
-  }
-
-  #[test]
-  fn length() {
-    assert_eq!(random_prefix_crockford_token("w", 10, false).unwrap().len(), 10);
-    assert_eq!(random_prefix_crockford_token("w:", 10, false).unwrap().len(), 10);
-    assert_eq!(random_prefix_crockford_token("prefix", 16, false).unwrap().len(), 16);
-    assert_eq!(random_prefix_crockford_token("prefix:", 16, false).unwrap().len(), 16);
-    assert_eq!(random_prefix_crockford_token("asdf:", 32, false).unwrap().len(), 32);
-    assert_eq!(random_prefix_crockford_token("asdf", 32, false).unwrap().len(), 32);
-    assert_eq!(random_prefix_crockford_token("w", 10, true).unwrap().len(), 10);
-    assert_eq!(random_prefix_crockford_token("w:", 10, true).unwrap().len(), 10);
-    assert_eq!(random_prefix_crockford_token("prefix", 16, true).unwrap().len(), 16);
-    assert_eq!(random_prefix_crockford_token("prefix:", 16, true).unwrap().len(), 16);
-    assert_eq!(random_prefix_crockford_token("asdf:", 32, true).unwrap().len(), 32);
-    assert_eq!(random_prefix_crockford_token("asdf", 32, true).unwrap().len(), 32);
-  }
-}
@@ -18,7 +18,7 @@ storyteller_root = { path = "../storyteller_root" }

 # External
 dotenv = "0.15.0"
-env_logger = "0.10.0"
+env_logger = "0.10.1"
 log = "0.4.14"
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

@@ -17,6 +17,7 @@ path = "src/lib.rs"
 # Internal
 errors = { path = "../../errors" }
 my-workspace-hack = { version = "0.1", path = "../../../../my-workspace-hack" }
+log = "0.4.20"

 # External
 # KEEP THIS TO A MINIMUM OR NIL.
@@ -0,0 +1,15 @@
+use std::path::Path;
+
+use errors::{AnyhowResult, bail};
+
+pub fn check_directory_exists<P: AsRef<Path>>(path: P) -> AnyhowResult<()> {
+  let check_path = path.as_ref();
+
+  if !check_path.exists() {
+    bail!("Path doesn't exist: {:?}", check_path);
+  }
+  if !check_path.is_dir() {
+    bail!("Path isn't a directory: {:?}", check_path);
+  }
+  Ok(())
+}
@@ -1,8 +1,7 @@
 use std::path::PathBuf;

-use anyhow::bail;
-
-use crate::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
+use errors::bail;

 pub fn check_file_exists(path: &PathBuf) -> AnyhowResult<()> {
  if !path.exists() {
@@ -7,6 +7,7 @@ use errors::{anyhow, AnyhowResult};
 pub fn file_size<P: AsRef<Path>>(filename: P) -> AnyhowResult<u64> {
  let metadata = std::fs::metadata(filename.as_ref())?;

+  // TODO(bt,2023-11-29): Return io::Result and errors here.
  if !metadata.is_file() {
    Err(anyhow!("path {:?} is not a file", filename.as_ref()))
  } else if metadata.is_symlink() {
@@ -0,0 +1,16 @@
+
+// Nearly a third of `ErrorKind` is nightly rust, so we're going to have to match on
+// unix error codes until the rest of the ErrorKind enum variants are stable.
+// The error codes are maintained in sys/unix/mod.rs - decode_error_kind(i32)
+
+const E_FILESYSTEM_QUOTA_EXCEEDED : i32 = 122; // pub const EDQUOT: ::c_int = 122;
+const E_STORAGE_FULL : i32 = 28; // pub const ENOSPC: ::c_int = 28;
+
+/// Reports if an io::Error was caused by the filesystem being full.
+pub fn is_filesystem_full_error(err: &std::io::Error) -> bool {
+  match err.raw_os_error() {
+    Some(E_FILESYSTEM_QUOTA_EXCEEDED) => true,
+    Some(E_STORAGE_FULL) => true,
+    _ => false,
+  }
+}
@@ -18,11 +18,17 @@
 #![allow(dead_code)]
 #![allow(non_snake_case)]

+pub mod check_file_exists;
 pub mod create_dir_all_if_missing;
 pub mod directory_exists;
 pub mod file_exists;
 pub mod file_read_bytes;
 pub mod file_size;
 pub mod filename_concat;
+pub mod is_filesystem_full_error;
 pub mod path_to_string;
 pub mod rename_across_devices;
+pub mod check_directory_exists;
+pub mod safe_delete_temp_file;
+pub mod safe_delete_possible_temp_file;
+pub mod safe_delete_temp_directory;
@@ -1,30 +1,80 @@
+use std::error::Error;
+use std::fmt;
 use std::path::Path;

+use log::error;
+
+use crate::is_filesystem_full_error::is_filesystem_full_error;
+
+#[derive(Debug)]
+pub enum RenameError {
+  StorageFull,
+  IoError(std::io::Error),
+}
+
+impl Error for RenameError {}
+
+impl fmt::Display for RenameError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      RenameError::StorageFull => write!(f, "RenameError: storage is full"),
+      RenameError::IoError(err) => write!(f, "{:?}", err),
+    }
+  }
+}
+
+impl From<std::io::Error> for RenameError {
+  fn from(error: std::io::Error) -> Self {
+    RenameError::IoError(error)
+  }
+}
+
 /// Rename a file.
 /// Ordinary rename will fail in Linux if it is across physical devices.
 /// This function will perform a copy followed by delete in that case.
-pub fn rename_across_devices<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> std::io::Result<()> {
+/// In both cases, this function will overwrite the destination if it already exists.
+pub fn rename_across_devices<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> Result<(), RenameError> {
  let result = std::fs::rename(&from, &to);

-  let err = match result.as_ref() {
+  let err = match result {
    Ok(_) => return Ok(()), // Rename succeeded.
    Err(err) => err,
  };

+  // Nearly a third of `ErrorKind` is nightly rust, so we're going to have to match on
+  // unix error codes until the rest of the ErrorKind enum variants are stable.
+  // The error codes are maintained in sys/unix/mod.rs - decode_error_kind(i32)
+  const E_CROSSES_DEVICES : i32 = 18; // pub const EXDEV: ::c_int = 18;
+
+  if is_filesystem_full_error(&err) {
+    error!("Filesystem is full during rename.");
+    return Err(RenameError::StorageFull);
+  }
+
  match err.raw_os_error() {
-    Some(18) => {
-      // NO-OP: pass-through
-      // NB: We can clean this up in the future when `err.kind() == ErrorKind::CrossDevices`
-      // stabilizes. For now we can assume code "18" means we're trying to rename across devices.
+    Some(E_CROSSES_DEVICES) => {
+      // NB: Fall through.
    }
    _ => {
      // Something else happened. Return original error.
-      return result;
+      return Err(RenameError::IoError(err));
    }
  }

-  let _num_bytes = std::fs::copy(&from, &to)?;
+  // NB: In production we've seen std::fs::copy silently succeed, yet copy zero bytes. The reason
+  // is more than likely that the filesystem is full. We'll check the number of copied bytes to
+  // confirm. Moreover, the Rust docs say wrt std::fs::copy, "On success, the total number of bytes
+  // copied is returned and it is equal to the length of the to file as reported by metadata."
+  let original_file_size = std::fs::metadata(from.as_ref())?.len();

+  let num_bytes_copied = std::fs::copy(&from, &to)?;
+
+  if num_bytes_copied != original_file_size {
+    error!("Filesystem is full during copy. Not all bytes copied.");
+    return Err(RenameError::StorageFull);
+  }
+
+  // An emulated "rename" would delete the source file.
  std::fs::remove_file(&from)?;

  Ok(())
@@ -1,6 +1,6 @@
 use std::path::Path;

-use crate::filesystem::safe_delete_temp_file::safe_delete_temp_file;
+use crate::safe_delete_temp_file::safe_delete_temp_file;

 pub fn safe_delete_possible_temp_file<P: AsRef<Path>>(maybe_file_path: Option<P>) {
  if let Some(file_path) = maybe_file_path {
@@ -15,7 +15,7 @@ path = "src/lib.rs"

 [dependencies]
 dotenv = "0.15.0"
-env_logger = "0.10.0"
+env_logger = "0.10.1"
 log = "0.4.20"
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

@@ -24,6 +24,8 @@ pub struct InsertGenericInferenceArgs<'a> {

  pub maybe_raw_inference_text: Option<&'a str>,

+  pub maybe_max_duration_seconds: Option<i32>,
+
  pub maybe_inference_args: Option<GenericInferenceArgs>,

  pub maybe_creator_user_token: Option<&'a UserToken>,
@@ -54,6 +56,11 @@ pub async fn insert_generic_inference_job(args: InsertGenericInferenceArgs<'_>)
        routing_tag
      });

+  // This only applies to certain types of inference.
+  // "0" is the default value, typically 12 seconds for TTS.
+  // "-1" means "unlimited"
+  let max_duration_seconds = args.maybe_max_duration_seconds.unwrap_or(0);
+
  let query = sqlx::query!(
        r#"
 INSERT INTO generic_inference_jobs
@@ -80,6 +87,7 @@ SET

  priority_level = ?,
  is_keepalive_required = ?,
+  max_duration_seconds = ?,

  is_debug_request = ?,
  maybe_routing_tag = ?,
@@ -109,6 +117,7 @@ SET

        args.priority_level,
        args.requires_keepalive,
+        max_duration_seconds,

        args.is_debug_request,
        maybe_routing_tag,
@@ -0,0 +1,157 @@
+use std::collections::HashSet;
+
+use anyhow::anyhow;
+use sqlx::{MySql, MySqlPool, QueryBuilder};
+
+use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
+use enums::common::job_status::JobStatus;
+use errors::AnyhowResult;
+
+/// Only certain job statuses should be modified.
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+pub enum KillableStatus {
+  Failed,
+  Pending,
+  Started,
+}
+
+/// Target everything, by job category, or by model type.
+#[derive(Clone)]
+pub enum KillableTarget {
+  AllJobs,
+  Category(InferenceCategory),
+  ModelType(InferenceModelType),
+}
+
+pub struct KillGenericInferenceJobsArgs<'a> {
+  pub job_statuses: HashSet<KillableStatus>,
+  pub target: KillableTarget,
+  pub maybe_priority_or_lower: Option<u8>,
+  pub mysql_pool: &'a MySqlPool,
+}
+
+pub async fn kill_generic_inference_jobs(args: KillGenericInferenceJobsArgs<'_>) -> AnyhowResult<()> {
+  let mut query = query_builder(
+    args.job_statuses,
+    args.target,
+    args.maybe_priority_or_lower
+  );
+
+  let query = query.build();
+
+  let result = query.execute(args.mysql_pool).await;
+
+  match result {
+    Err(err) => {
+      Err(anyhow!("error with query: {:?}", err))
+    },
+    Ok(_r) => Ok(()),
+  }
+}
+
+fn query_builder(
+  job_statuses: HashSet<KillableStatus>,
+  target: KillableTarget,
+  maybe_priority_or_lower: Option<u8>,
+) -> QueryBuilder<'static, MySql> {
+
+  // NB: Query cannot be statically checked by sqlx
+  let mut query_builder: QueryBuilder<MySql> = QueryBuilder::new(
+    r#"
+UPDATE generic_inference_jobs
+SET status='cancelled_by_system'
+WHERE status IN(
+    "#
+  );
+
+  let job_statuses = job_statuses.into_iter()
+      .map(|status| match status {
+        KillableStatus::Failed => JobStatus::AttemptFailed.to_str(),
+        KillableStatus::Pending => JobStatus::Pending.to_str(),
+        KillableStatus::Started => JobStatus::Started.to_str(),
+      })
+      .collect::<Vec<_>>();
+
+  // NB: Syntax will be wrong if list has zero length
+  let mut separated = query_builder.separated(", ");
+  for value_type in job_statuses.into_iter() {
+    separated.push_bind(value_type);
+  }
+  separated.push_unseparated(") ");
+
+  if let Some(priority) = maybe_priority_or_lower {
+    query_builder.push(format!(" AND priority_level <= {priority} "));
+  }
+
+  match target {
+    KillableTarget::AllJobs => {
+      // No need to modify query.
+    }
+    KillableTarget::Category(category) => {
+      query_builder.push(" AND inference_category = ");
+      query_builder.push_bind(category);
+    }
+    KillableTarget::ModelType(model_type) => {
+      query_builder.push(" AND maybe_model_type = ");
+      query_builder.push_bind(model_type);
+    }
+  }
+
+  query_builder
+}
+
+#[cfg(test)]
+mod tests {
+  use std::collections::HashSet;
+
+  use regex::Regex;
+
+  use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+  use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
+
+  use crate::queries::generic_inference::web::kill_generic_inference_jobs::{KillableStatus, KillableTarget, query_builder};
+
+  fn cleanup_query(query: &str) -> String {
+    let regex = Regex::new(r#"\s+"#).unwrap();
+    let result = regex.replace_all(query, " ").trim().to_string();
+    result
+  }
+
+  #[test]
+  fn test_statuses() {
+    let statuses = HashSet::from([KillableStatus::Failed]);
+    let builder = query_builder(statuses, KillableTarget::AllJobs, None);
+    let query = cleanup_query(&builder.into_sql());
+    assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?)".to_string());
+
+    let statuses = HashSet::from([KillableStatus::Failed, KillableStatus::Pending, KillableStatus::Started]);
+    let builder = query_builder(statuses, KillableTarget::AllJobs, None);
+    let query = cleanup_query(&builder.into_sql());
+    assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?, ?, ?)".to_string());
+  }
+
+  #[test]
+  fn test_priority() {
+    let statuses = HashSet::from([KillableStatus::Failed]);
+    let builder = query_builder(statuses, KillableTarget::AllJobs, Some(10));
+    let query = cleanup_query(&builder.into_sql());
+    assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND priority_level <= 10".to_string());
+  }
+
+  #[test]
+  fn test_category_target() {
+    let statuses = HashSet::from([KillableStatus::Failed]);
+    let builder = query_builder(statuses, KillableTarget::Category(InferenceCategory::LipsyncAnimation), None);
+    let query = cleanup_query(&builder.into_sql());
+    assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND inference_category = ?".to_string());
+  }
+
+  #[test]
+  fn test_model_target() {
+    let statuses = HashSet::from([KillableStatus::Failed]);
+    let builder = query_builder(statuses, KillableTarget::ModelType(InferenceModelType::RvcV2), None);
+    let query = cleanup_query(&builder.into_sql());
+    assert_eq!(query, "UPDATE generic_inference_jobs SET status='cancelled_by_system' WHERE status IN( ?) AND maybe_model_type = ?".to_string());
+  }
+}
@@ -1,3 +1,4 @@
 pub mod get_inference_job_status;
 pub mod get_pending_inference_job_count;
 pub mod insert_generic_inference_job;
+pub mod kill_generic_inference_jobs;
@@ -157,7 +157,7 @@ async fn select_include_deleted(
            "#,
            weight_token.as_str()
        )
-        .fetch_one(mysql_connection).await
+        .fetch_one(&mut **mysql_connection).await
 }

 async fn select_without_deleted(
@@ -207,7 +207,7 @@ async fn select_without_deleted(
        "#,
            weight_token.as_str()
        )
-        .fetch_one(mysql_connection).await
+        .fetch_one(&mut **mysql_connection).await
 }

 // RawWeight is the struct that is returned from the database in raw form.
@@ -1,19 +1,13 @@
+use chrono::{DateTime, Utc};
+use sqlx::{Acquire, MySqlConnection, MySqlPool};

-use anyhow::anyhow;
-use chrono::{ DateTime, Utc };
-use enums::by_table::model_weights::weights_types;
-use log::{ info, warn };
-use sqlx::{ MySql, MySqlPool, query, Acquire, MySqlConnection };
-use sqlx::pool::PoolConnection;
-
+use enums::by_table::model_weights::{
+  weights_category::WeightsCategory,
+  weights_types::WeightsType,
+};
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use tokens::tokens::model_weights::ModelWeightToken;
-
-use enums::by_table::model_weights::{
-    weights_types::WeightsType,
-    weights_category::WeightsCategory,
-};
 use tokens::tokens::users::UserToken;

 #[derive(Serialize)]
@@ -136,7 +130,7 @@ async fn get_raw_weights_by_creator_username(
            "#,
            creator_username).fetch_all(connection)
        .await?;
-        return Ok(raw_weights);
+        Ok(raw_weights)
    } else {
        let raw_weights: Vec<RawWeightJoinUser> = sqlx::query_as!(
            RawWeightJoinUser,
@@ -1,15 +1,10 @@
 use anyhow::anyhow;
 use sqlx::MySqlPool;

-use log::info;
-use sqlx::mysql::MySqlPoolOptions;
-
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use tokens::tokens::model_weights::ModelWeightToken;

-use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
-
 pub struct UpdateWeightArgs<'a> {
    pub weight_token: &'a ModelWeightToken,
    pub title: Option<&'a str>,
@@ -58,13 +53,13 @@ pub async fn update_weights(args: UpdateWeightArgs<'_>) -> AnyhowResult<()> {

 #[cfg(test)]
 mod tests {
-
-    use tokio;
-    // Template 
+    // Template
    use sqlx::mysql::MySqlPoolOptions;
+    use tokio;

-    use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING};
+    use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
    use errors::AnyhowResult;
+
    #[tokio::test]
    async fn test_update_weights() -> AnyhowResult<()> {
        let db_connection_string = DEFAULT_MYSQL_CONNECTION_STRING;
@@ -26,5 +26,5 @@ serde_derive = "1.0.125"
 serde_json = "1.0.108"

 # TODO: Get rid of in favor of std::time
-time = "0.2.26"
+time = "0.3.30"
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
@@ -30,7 +30,7 @@ chrono = { version = "=0.4.22", features = ["serde"] }
 dotenv = "0.15.0"
 hostname = "0.3.1"
 log = "0.4.14"
-rand = "0.8.3"
+rand = "0.8.5"

 # MySql Database
 # "offline" supports typesafe builds without a live database
@@ -6,7 +6,7 @@ use tempdir::TempDir;

 use config::is_bad_download_url::is_bad_download_url;
 use container_common::anyhow_result::AnyhowResult;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
 use mysql_queries::queries::generic_download::job::mark_generic_download_job_done::mark_generic_download_job_done;
@@ -6,12 +6,12 @@ use anyhow::anyhow;
 use log::{info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::tts_models::tts_model_type::TtsModelType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
@@ -4,14 +4,14 @@ use anyhow::anyhow;
 use log::{error, info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::tts_models::tts_model_type::TtsModelType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
 use filesys::file_size::file_size;
 use filesys::filename_concat::filename_concat;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
@@ -6,11 +6,11 @@ use anyhow::anyhow;
 use log::{info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::common::vocoder_type::VocoderType;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
@@ -6,11 +6,11 @@ use anyhow::anyhow;
 use log::{info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::common::vocoder_type::VocoderType;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
@@ -3,8 +3,8 @@ use std::path::{Path, PathBuf};
 use log::{info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use errors::AnyhowResult;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;

 #[derive(Debug)]
@@ -5,10 +5,10 @@ use log::info;
 use tempdir::TempDir;

 use cloud_storage::bucket_client::BucketClient;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
 use errors::AnyhowResult;
 use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
 use filesys::file_exists::file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;

 #[derive(Clone)]
 pub struct PretrainedHubertModel {
@@ -4,15 +4,15 @@ use anyhow::anyhow;
 use log::{error, info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_possible_temp_file::safe_delete_possible_temp_file;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use crockford::crockford_entropy_lower;
 use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
 use filesys::file_size::file_size;
+use filesys::safe_delete_possible_temp_file::safe_delete_possible_temp_file;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
 use mysql_queries::queries::voice_conversion::models::insert_voice_conversion_model_from_download_job::{insert_voice_conversion_model_from_download_job, InsertVoiceConversionModelArgs};
@@ -4,13 +4,13 @@ use anyhow::anyhow;
 use log::{error, info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
 use filesys::file_size::file_size;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
@@ -6,12 +6,12 @@ use anyhow::anyhow;
 use log::{info, warn};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use mysql_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
@@ -33,8 +33,8 @@ use cloud_storage::bucket_path_unifier::BucketPathUnifier;
 use config::common_env::CommonEnv;
 use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
-use container_common::filesystem::check_directory_exists::check_directory_exists;
 use errors::AnyhowResult;
+use filesys::check_directory_exists::check_directory_exists;
 use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
 use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg;
 use mysql_queries::mediators::badge_granter::BadgeGranter;
@@ -19,9 +19,9 @@ actix_helpers = { path = "../../../lib/actix_helpers" }
 bootstrap = { path = "../../../lib/bootstrap" }
 buckets = { path = "../../../_types/buckets" }
 cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
+collections = { path = "../../../lib/collections" }
 concurrency = { path = "../../../lib/concurrency" }
 config = { path = "../../../lib/deprecated/config" }
-container_common = { path = "../../../lib/deprecated/container_common" }
 easyenv = { path = "../../../lib/easyenv" }
 enums = { path = "../../../_types/enums" }
 errors = { path = "../../../lib/errors" }
@@ -46,6 +46,7 @@ anyhow = "1.0.75"
 chrono = { version = "=0.4.22", features = ["serde"] }
 dotenv = "0.15.0"
 hostname = "0.3.1"
+lfu = "0.2.5"
 log = "0.4.14"
 once_cell = "1.18.0"
 subprocess = "0.2.9"
@@ -1,16 +1,25 @@
+use std::path::Path;
+
 use log::warn;

-use container_common::collections::multiple_random_from_vec::multiple_random_from_vec;
+use collections::multiple_random_from_vec::multiple_random_from_vec;
 use errors::AnyhowResult;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;

 pub fn clear_full_filesystem(cache_dir: &SemiPersistentCacheDir) -> AnyhowResult<()> {
-  warn!("Deleting cached VC models to free up disk space.");
+  warn!("Deleting cached models to free up disk space.");

-  let model_dir = cache_dir.voice_conversion_model_directory();
+  decimate_directory(cache_dir.custom_vocoder_model_directory())?;
+  decimate_directory(cache_dir.tts_synthesizer_model_directory())?;
+  decimate_directory(cache_dir.voice_conversion_model_directory())?;

+  Ok(())
+}
+
+fn decimate_directory(path: &Path) -> AnyhowResult<()> {
  // TODO: When this is no longer sufficient, delete other types of locally-cached data.
-  let paths = std::fs::read_dir(model_dir)?
+  let paths = std::fs::read_dir(path)?
      .map(|res| res.map(|e| e.path()))
      .collect::<Result<Vec<_>, std::io::Error>>()?;

@@ -21,8 +30,8 @@ pub fn clear_full_filesystem(cache_dir: &SemiPersistentCacheDir) -> AnyhowResult
  for model_to_delete in models_to_delete {
    warn!("Deleting cached model file: {:?}", model_to_delete);

-    let full_model_path = cache_dir.voice_conversion_model_path(model_to_delete);
-    std::fs::remove_file(full_model_path)?;
+    let full_model_path = path.join(model_to_delete);
+    safe_delete_temp_file(full_model_path);
  }

  Ok(())
@@ -1,16 +1,17 @@
 use std::path::PathBuf;
+
 use anyhow::anyhow;
 use log::{info, warn};

 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use enums::by_table::tts_models::tts_model_type::TtsModelType;
 use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
-use errors::AnyhowResult;
 use filesys::file_exists::file_exists;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceRecord};
+use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord};
 use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference};

+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job_dependencies::JobDependencies;

 pub struct DependencyStatus {
@@ -31,7 +32,7 @@ pub enum MaybeInferenceModel {
  VcModel(VoiceConversionModelForInference)
 }

-pub async fn determine_dependency_status(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> AnyhowResult<DependencyStatus> {
+pub async fn determine_dependency_status(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<DependencyStatus, ProcessSingleJobError> {
  let maybe_model = get_model_record_from_cacheable_query(job_dependencies, job).await?;
  let maybe_token_and_path = get_model_token_and_path(job_dependencies, &maybe_model);

@@ -105,10 +106,16 @@ fn get_model_token_and_path(job_dependencies: &JobDependencies, maybe_model: &Ma
  }
 }

-async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> AnyhowResult<MaybeInferenceModel> {
+async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<MaybeInferenceModel, ProcessSingleJobError> {
  let model = match (job.inference_category, job.maybe_model_token.as_deref()) {
    (InferenceCategory::TextToSpeech, Some(token)) => {
-      let maybe_model = job_dependencies.job.info.caches.tts_model_record_cache.copy_without_bump_if_unexpired(token.to_string())?;
+      let maybe_model = job_dependencies
+          .job
+          .info
+          .caches
+          .tts_model_record_cache
+          .copy_without_bump_if_unexpired(token.to_string())
+          .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;

      match maybe_model {
        Some(model) => MaybeInferenceModel::TtsModel(model),
@@ -116,22 +123,36 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie
          let maybe_tts_model = get_tts_model_for_inference_improved(
            &job_dependencies.db.mysql_pool, token)
              .await
-              .map_err(|err| anyhow!("database error: {:?}", err))?;
+              .map_err(|err| match err {
+                TtsModelForInferenceError::ModelDeleted => ProcessSingleJobError::ModelDeleted,
+                _ => ProcessSingleJobError::Other(anyhow!("database error: {:?}", err))
+              })?;

          match maybe_tts_model {
+            None => MaybeInferenceModel::None,
            Some(model) => {
              let token = token.to_string();
-              let _r = job_dependencies.job.info.caches.tts_model_record_cache.store_copy(&token, &model)?;
+              let _r = job_dependencies
+                  .job
+                  .info
+                  .caches
+                  .tts_model_record_cache
+                  .store_copy(&token, &model)
+                  .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
              MaybeInferenceModel::TtsModel(model)
            },
-
-            None => MaybeInferenceModel::None,
          }
        }
      }
    }
    (InferenceCategory::VoiceConversion, Some(token)) => {
-      let maybe_model = job_dependencies.job.info.caches.vc_model_record_cache.copy_without_bump_if_unexpired(token.to_string())?;
+      let maybe_model = job_dependencies
+          .job
+          .info
+          .caches
+          .vc_model_record_cache
+          .copy_without_bump_if_unexpired(token.to_string())
+          .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;

      match maybe_model {
        Some(model) => MaybeInferenceModel::VcModel(model),
@@ -139,16 +160,22 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie
          let maybe_vc_model = get_voice_conversion_model_for_inference(
            &job_dependencies.db.mysql_pool, token)
              .await
-              .map_err(|err| anyhow!("database error: {:?}", err))?;
+              .map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?;

          match maybe_vc_model {
+            None => MaybeInferenceModel::None,
            Some(model) => {
              let token = token.to_string();
-              let _r = job_dependencies.job.info.caches.vc_model_record_cache.store_copy(&token, &model)?;
+              let _r = job_dependencies
+                  .job
+                  .info
+                  .caches
+                  .vc_model_record_cache
+                  .store_copy(&token, &model)
+                  .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+
              MaybeInferenceModel::VcModel(model)
            },
-
-            None => MaybeInferenceModel::None,
          }
        }
      }
@@ -1,4 +1,4 @@
-use std::time::Duration;
+use std::time::{Duration, Instant};

 use log::{error, info, warn};

@@ -23,6 +23,9 @@ const INCREASE_TIMEOUT_MILLIS : u64 = 1000;
 /// Pause file millis
 const PAUSE_FILE_EXISTS_WAIT_MILLIS : u64 = 1000 * 30;

+/// Warn on slow batch queries
+const SLOW_BATCH_QUERY_NOTICE_DURATION : Duration = Duration::from_millis(3500);
+
 pub async fn main_loop(job_dependencies: JobDependencies) {
  let mut noop_logger = NoOpLogger::new(job_dependencies.job.system.no_op_logger_millis as i64);

@@ -46,6 +49,8 @@ pub async fn main_loop(job_dependencies: JobDependencies) {

    let maybe_scoped_model_types = job_dependencies.job.system.scoped_execution.get_scoped_model_types();

+    let batch_query_start_time = Instant::now();
+
    let maybe_available_jobs = list_available_generic_inference_jobs(ListAvailableGenericInferenceJobArgs {
      num_records: job_dependencies.job.system.job_batch_size,
      is_debug_worker: false, // TODO
@@ -55,6 +60,12 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
      mysql_pool: &job_dependencies.db.mysql_pool,
    }).await;

+    let batch_query_duration = Instant::now().duration_since(batch_query_start_time);
+
+    if batch_query_duration > SLOW_BATCH_QUERY_NOTICE_DURATION {
+      warn!("Batch query took duration: {:?}", &batch_query_duration);
+    }
+
    sort_by_priority = true;
    sort_by_priority_count += 1;

@@ -106,11 +117,13 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
 //  This probably returns a struct or enum with some measure of how many GB need to be downloaded.

 async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<AvailableInferenceJob>) -> AnyhowResult<()> {
-  for job in jobs.into_iter() {
+  let job_count = jobs.len();
+  for (i, job) in jobs.into_iter().enumerate() {
    let result = process_single_job(job_dependencies, &job).await;
+
    match result {
      Ok(success_case) => {
-        info!("Job loop iteration \"success\": {:?}", success_case);
+        info!("Job loop iteration ({i} of {job_count} batch) \"success\": {:?}", success_case);

        let increment_success_count = match success_case {
          ProcessSingleJobSuccessCase::JobCompleted => true,
@@ -125,7 +138,10 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
        }
      },
      Err(err) => {
-        warn!("Failure to process job: {:?}", err);
+        error!(
+          r#"Failure to process job ({i} of {job_count} batch): {:?} -
+            {:?}
+          "#,job.inference_job_token, err);
        let _r = handle_error(&job_dependencies, &job, err).await?;
      }
    }
@@ -191,7 +207,14 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere
        None,
        Some(FrontendFailureCategory::FaceNotDetected),
      ),
-
+    ProcessSingleJobError::ModelDeleted =>
+      (
+        JobFailureClass::PermanentFailure,
+        ContainerHealth::Ignore,
+        "model deleted".to_string(),
+        None,
+        None,
+      ),
    // Non-permanent failures
    ProcessSingleJobError::FilesystemFull =>
      (
@@ -209,7 +232,14 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere
        None, // Obviously don't tell the user about errors even we're not sure about
        Some(FrontendFailureCategory::RetryableWorkerError),
      ),
-
+    ProcessSingleJobError::IoError(ref err) =>
+      (
+        JobFailureClass::TransientFailure,
+        ContainerHealth::IncrementContainerFailCount,
+        format!("IoError: {:?}", err),
+        None, // Obviously don't tell the user about errors even we're not sure about
+        Some(FrontendFailureCategory::RetryableWorkerError),
+      ),
    ProcessSingleJobError::JobSystemMisconfiguration(ref maybe_reason) =>
      (
        JobFailureClass::TransientFailure,
@@ -261,6 +291,8 @@ async fn handle_error(job_dependencies: &&JobDependencies, job: &AvailableInfere
    ProcessSingleJobError::NotYetImplemented => {}
    ProcessSingleJobError::FaceDetectionFailure => {}
    ProcessSingleJobError::JobSystemMisconfiguration(_) => {}
+    ProcessSingleJobError::ModelDeleted => {}
+    ProcessSingleJobError::IoError(_) => {}
  }

  Ok(())
@@ -11,7 +11,6 @@ use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_s
 use redis_common::redis_keys::RedisKeys;

 use crate::job::job_loop::determine_dependency_status::determine_dependency_status;
-use crate::job::job_loop::job_success_result::ResultEntity;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
 use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
@@ -40,15 +39,14 @@ pub async fn process_single_job(
    }
  }

-  // TODO(bt,2023-07-23): Re-review the following. It looks sus.
-  // TODO(bt,2023-07-23): Re-review the following. It looks sus.
-  // TODO(bt,2023-07-23): Re-review the following. It looks sus.
  // TODO(bt,2023-07-23): Re-review the following. It looks sus.
  let dependency_status = determine_dependency_status(job_dependencies, job)
-      .await
-      .map_err(|err| ProcessSingleJobError::Other(anyhow!("database or cache error: {:?}", err)))?;
+      .await?;

-  if !force_execution && !dependency_status.models_already_on_filesystem {
+  if !force_execution
+      && !job_dependencies.job.system.always_allow_cold_filesystem_cache
+      && !dependency_status.models_already_on_filesystem
+  {
    match dependency_status.maybe_model_token {
      None => {} // No model token, proceed
      Some(model_token) => {
@@ -93,7 +91,7 @@ async fn process_single_job_wrap_with_logs(

  println!("\n  ----------------------------------------- JOB START -----------------------------------------  \n");

-  info!("Beginning work on job ({}): {:?}", job.id.0, job.inference_job_token);
+  info!("Lock acquired. Beginning work on job ({}): {:?}", job.id.0, job.inference_job_token);
  info!("Job category: {:?}", job.inference_category);
  info!("Job model type: {:?}", job.maybe_model_type);
  info!("Job model token: {:?}", job.maybe_model_token);
@@ -161,18 +159,19 @@ async fn do_process_single_job(
    }
  }

-  // ==================== SETUP TEMP DIRS ==================== //
-
-  let temp_dir = format!("temp_{}", job.id.0);
-  let temp_dir = job_dependencies.fs.scoped_temp_dir_creator_for_downloads.new_tempdir(&temp_dir)
-      .map_err(|err| ProcessSingleJobError::Other(anyhow!("filesystem error: {:?}", err)))?;
-
-  let _p = temp_dir.path(); // TODO: Just so the build doesn't complain about unused. Remove.
+// NB(bt,2023-11-29): This looks dead, but hopefully isn't leaned on for downstream side effects.
+// I'll leave this commented out while dealing with the outage, but this can probably be removed
+// in a short while after we redeploy and verify all the jobs.
+//  // ==================== SETUP TEMP DIRS ==================== //
+//
+//  let temp_dir = format!("temp_{}", job.id.0);
+//  let temp_dir = job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads.new_tempdir(&temp_dir)
+//      .map_err(|err| ProcessSingleJobError::Other(anyhow!("filesystem error: {:?}", err)))?;
+//
+//  let _p = temp_dir.path(); // TODO: Just so the build doesn't complain about unused. Remove.

  // ==================== HANDLE DIFFERENT INFERENCE TYPES ==================== //

-  let mut maybe_result_entity : Option<ResultEntity>;
-
  let job_success_result = match job.inference_category {
    InferenceCategory::LipsyncAnimation => {
      process_single_lipsync_job(job_dependencies, job).await?
@@ -1,4 +1,4 @@
-use anyhow::anyhow;
+use filesys::is_filesystem_full_error::is_filesystem_full_error;

 /// Error from processing a single job
 #[derive(Debug)]
@@ -25,6 +25,13 @@ pub enum ProcessSingleJobError {
  /// It might be important, we just haven't special cased it yet.
  Other(anyhow::Error),

+  /// This is a non-special-cased IO Error
+  /// It might be important, we just haven't special cased it yet.
+  IoError(std::io::Error),
+
+  /// Job underlying resources are missing - model deleted
+  ModelDeleted,
+
  /// The job system is misconfigured.
  /// Retry the job, but mark as a job runner system failure (health check failure).
  JobSystemMisconfiguration(Option<String>),
@@ -34,25 +41,37 @@ pub enum ProcessSingleJobError {
  NotYetImplemented
 }

+impl From<std::io::Error> for ProcessSingleJobError {
+  fn from(error: std::io::Error) -> Self {
+    ProcessSingleJobError::from_io_error(error)
+  }
+}
+
+impl From<anyhow::Error> for ProcessSingleJobError {
+  fn from(error: anyhow::Error) -> Self {
+    ProcessSingleJobError::from_anyhow_error(error)
+  }
+}
+
 impl ProcessSingleJobError {
  pub fn from_io_error(error: std::io::Error) -> Self {
-    match error.raw_os_error() {
-      // NB: We can't use err.kind() -> ErrorKind::StorageFull, because it's marked unstable:
-      // `io_error_more` is unstable [E0658]
-      Some(28) => ProcessSingleJobError::FilesystemFull,
-      _ => ProcessSingleJobError::Other(anyhow!(error)),
+    if is_filesystem_full_error(&error) {
+      ProcessSingleJobError::FilesystemFull
+    } else {
+      ProcessSingleJobError::IoError(error)
    }
  }

  pub fn from_anyhow_error(error: anyhow::Error) -> Self {
-    match error.downcast_ref::<std::io::Error>() {
-      Some(e) => match e.raw_os_error() {
-        // NB: We can't use err.kind() -> ErrorKind::StorageFull, because it's marked unstable:
-        // `io_error_more` is unstable [E0658]
-        Some(28) => ProcessSingleJobError::FilesystemFull,
-        _ => ProcessSingleJobError::Other(anyhow!(error)),
+    match error.downcast::<std::io::Error>() {
+      Ok(io_error) => {
+        if is_filesystem_full_error(&io_error) {
+          ProcessSingleJobError::FilesystemFull
+        } else {
+          ProcessSingleJobError::IoError(io_error)
+        }
      },
-      None => ProcessSingleJobError::Other(error),
+      Err(anyhow_error) => ProcessSingleJobError::Other(anyhow_error),
    }
  }
 }
@@ -13,7 +13,9 @@ pub fn categorize_error(stderr_contents: &str) -> Option<ProcessSingleJobError>
 #[cfg(test)]
 mod tests {
  use anyhow::bail;
-  use container_common::anyhow_result::AnyhowResult;
+
+  use errors::AnyhowResult;
+
  use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
  use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error;

@@ -16,7 +16,7 @@ use mysql_queries::queries::voice_conversion::results::get_voice_conversion_resu
 use tokens::tokens::media_uploads::MediaUploadToken;

 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;

 pub struct AudioFile {
@@ -94,16 +94,17 @@ pub async fn download_audio_file(

  let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin");

-  maybe_download_file_from_bucket(
-    "audio file",
-    &downloaded_filesystem_path,
-    &bucket_object_path,
-    public_bucket_client,
+  maybe_download_file_from_bucket(MaybeDownloadArgs {
+    name_or_description_of_file: "audio file",
+    final_filesystem_file_path: &downloaded_filesystem_path,
+    bucket_object_path: &bucket_object_path,
+    bucket_client: public_bucket_client,
    job_progress_reporter,
-    "downloading",
-    job.id.0,
-    &temp_dir_creator,
-  ).await?;
+    job_progress_update_description: "downloading",
+    job_id: job.id.0,
+    scoped_tempdir_creator: &temp_dir_creator,
+    maybe_existing_file_minimum_size_required: None,
+  }).await?;

  Ok(AudioFile {
    filesystem_path: downloaded_filesystem_path,
@@ -14,7 +14,7 @@ use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_m
 use tokens::tokens::media_uploads::MediaUploadToken;

 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;

 pub struct ImageFile {
@@ -65,16 +65,17 @@ pub async fn download_image_file(

  let downloaded_filesystem_path = work_temp_dir.path().join("image.bin");

-  maybe_download_file_from_bucket(
-    "image file",
-    &downloaded_filesystem_path,
-    &bucket_object_path,
-    public_bucket_client,
+  maybe_download_file_from_bucket(MaybeDownloadArgs {
+    name_or_description_of_file: "image file",
+    final_filesystem_file_path: &downloaded_filesystem_path,
+    bucket_object_path: &bucket_object_path,
+    bucket_client: public_bucket_client,
    job_progress_reporter,
-    "downloading",
-    job.id.0,
-    &temp_dir_creator,
-  ).await?;
+    job_progress_update_description: "downloading",
+    job_id: job.id.0,
+    scoped_tempdir_creator: &temp_dir_creator,
+    maybe_existing_file_minimum_size_required: None,
+  }).await?;

  Ok(ImageFile {
    filesystem_path: downloaded_filesystem_path,
@@ -5,11 +5,11 @@ use anyhow::anyhow;
 use log::{error, info, warn};

 use buckets::public::media_files::original_file::MediaFileBucketPath;
-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use filesys::check_file_exists::check_file_exists;
 use filesys::file_size::file_size;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -74,12 +74,12 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess

    let result = downloader.download_if_not_on_filesystem(
      &args.job_dependencies.buckets.private_bucket_client,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
+      &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
    ).await;

-    if let Err(e) = result {
-      error!("could not download: {:?}", e);
-      return Err(ProcessSingleJobError::from_anyhow_error(e))
+    if let Err(err) = result {
+      error!("could not download: {:?}", err);
+      return Err(err);
    }
  }

@@ -10,7 +10,7 @@ use log::info;
 use once_cell::sync::Lazy;
 use subprocess::{Popen, PopenConfig, Redirection};

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};

@@ -6,7 +6,7 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies;
 use crate::job_dependencies::JobDependencies;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};

 pub struct StaticDependencies {
  pub waveglow_vocoder_model_fs_path : PathBuf,
@@ -22,7 +22,6 @@ pub async fn download_static_dependencies<'a>(
  mut job_progress_reporter: &mut Box<dyn JobProgressReporter>,
 ) -> Result<StaticDependencies, ProcessSingleJobError> {

-
  // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== //

  let waveglow_vocoder_model_fs_path = {
@@ -31,16 +30,17 @@ pub async fn download_static_dependencies<'a>(
    let waveglow_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&waveglow_vocoder_model_filename);
    let waveglow_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename);

-    maybe_download_file_from_bucket(
-      "waveglow vocoder model",
-      &waveglow_vocoder_model_fs_path,
-      &waveglow_vocoder_model_object_path,
-      &job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading vocoder (1 of 3)",
-      job.id.0,
-      &job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "waveglow vocoder model",
+      final_filesystem_file_path: &waveglow_vocoder_model_fs_path,
+      bucket_object_path: &waveglow_vocoder_model_object_path,
+      bucket_client: &job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading vocoder (1 of 3)",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
+      maybe_existing_file_minimum_size_required: Some(1000),
+    }).await?;

    waveglow_vocoder_model_fs_path
  };
@@ -53,16 +53,17 @@ pub async fn download_static_dependencies<'a>(
    let hifigan_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_vocoder_model_filename);
    let hifigan_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename);

-    maybe_download_file_from_bucket(
-      "hifigan vocoder model",
-      &hifigan_vocoder_model_fs_path,
-      &hifigan_vocoder_model_object_path,
-      &job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading vocoder (2 of 3)",
-      job.id.0,
-      &job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "hifigan vocoder model",
+      final_filesystem_file_path: &hifigan_vocoder_model_fs_path,
+      bucket_object_path: &hifigan_vocoder_model_object_path,
+      bucket_client: &job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading vocoder (2 of 3)",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
+      maybe_existing_file_minimum_size_required: Some(1000),
+    }).await?;

    hifigan_vocoder_model_fs_path
  };
@@ -75,16 +76,17 @@ pub async fn download_static_dependencies<'a>(
    let hifigan_superres_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_superres_vocoder_model_filename);
    let hifigan_superres_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename);

-    maybe_download_file_from_bucket(
-      "hifigan superres vocoder model",
-      &hifigan_superres_vocoder_model_fs_path,
-      &hifigan_superres_vocoder_model_object_path,
-      &job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading vocoder (3 of 3)",
-      job.id.0,
-      &job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "hifigan superres vocoder model",
+      final_filesystem_file_path: &hifigan_superres_vocoder_model_fs_path,
+      bucket_object_path: &hifigan_superres_vocoder_model_object_path,
+      bucket_client: &job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading vocoder (3 of 3)",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
+      maybe_existing_file_minimum_size_required: Some(1000),
+    }).await?;

    hifigan_superres_vocoder_model_fs_path
  };
@@ -8,4 +8,5 @@ pub mod tacotron2_dependencies;
 pub mod tacotron2_inference_command;
 pub mod tacotron2_inference_sidecar_client;
 pub mod tacotron2_sidecar_health_check_client;
+pub mod virtual_lfu_cache;
 pub mod vocoder_option;
@@ -7,11 +7,11 @@ use anyhow::anyhow;
 use log::{error, info};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_string::sha256_hash_string;
 use mysql_queries::column_types::vocoder_type::VocoderType;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -29,7 +29,7 @@ use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_s
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor};
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption;
 use crate::job_dependencies::JobDependencies;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};

 /// Text starting with this will be treated as a test request.
 /// This allows the request to bypass the model cache and query the latest TTS model.
@@ -58,7 +58,7 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<JobSuccessResult, P

  // NB: The first time TT2 on inference-job was deployed, the filesystem filled up with
  // temporary directories. This is just being abundantly safe.
-  info!("Deleting temp directory: {:?}", work_temp_dir.path());
+  info!("(After job cleanup) Deleting temp directory: {:?}", work_temp_dir.path());
  safe_delete_temp_directory(&work_temp_dir);

  result
@@ -119,16 +119,17 @@ async fn process_job_with_cleanup(
      let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token);
      let custom_vocoder_object_path  = args.job_dependencies.buckets.bucket_path_unifier.vocoder_path(&vocoder.vocoder_private_bucket_hash);

-      maybe_download_file_from_bucket(
-        "custom vocoder",
-        &custom_vocoder_fs_path,
-        &custom_vocoder_object_path,
-        &args.job_dependencies.buckets.private_bucket_client,
-        &mut job_progress_reporter,
-        "downloading user vocoder",
-        job.id.0,
-        &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-      ).await?;
+      maybe_download_file_from_bucket(MaybeDownloadArgs {
+        name_or_description_of_file: "custom vocoder",
+        final_filesystem_file_path: &custom_vocoder_fs_path,
+        bucket_object_path: &custom_vocoder_object_path,
+        bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+        job_progress_reporter: &mut job_progress_reporter,
+        job_progress_update_description: "downloading user vocoder",
+        job_id: job.id.0,
+        scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+        maybe_existing_file_minimum_size_required: Some(1000),
+      }).await?;

      Some(custom_vocoder_fs_path)
    }
@@ -140,16 +141,17 @@ async fn process_job_with_cleanup(
    let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str());
    let tts_synthesizer_object_path  = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash);

-    maybe_download_file_from_bucket(
-      "synthesizer",
-      &tts_synthesizer_fs_path,
-      &tts_synthesizer_object_path,
-      &args.job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading synthesizer",
-      job.id.0,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "synthesizer",
+      final_filesystem_file_path: & tts_synthesizer_fs_path,
+      bucket_object_path: &tts_synthesizer_object_path,
+      bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading synthesizer",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      maybe_existing_file_minimum_size_required: Some(1000),
+    }).await?;

    tts_synthesizer_fs_path
  };
@@ -238,12 +240,20 @@ async fn process_job_with_cleanup(
    maybe_mel_multiply_factor = Some(MelMultiplyFactor::DefaultMultiplyFactor);
  }

-  // TODO(bt,2023-11-27): Handle LRU cache on python side
-  let maybe_unload_model_path = None;
+  let maybe_unload_model_path = model_dependencies
+      .sidecar
+      .virtual_lfu_cache
+      .insert_returning_replaced(tts_synthesizer_fs_path.to_str().unwrap_or(""))
+      .map_err(|e| ProcessSingleJobError::Other(e))?;
+
+  if let Some(model_path) = maybe_unload_model_path.as_deref() {
+    info!("Remove model from sidecar LFU cache: {:?}", model_path);
+  }

  let inference_start_time = Instant::now();

  if model_dependencies.sidecar.use_sidecar_instead_of_shell {
+    info!("Calling inference sidecar...");
    let _r = model_dependencies.sidecar.inference_client.request_inference(
      &cleaned_inference_text,
      max_decoder_steps,
@@ -258,6 +268,7 @@ async fn process_job_with_cleanup(
      tts_model.maybe_custom_mel_multiply_factor,
    ).await.map_err(|e| ProcessSingleJobError::Other(e))?;
  } else {
+    info!("Shelling out for inference...");
    let _r = model_dependencies.inference_command.execute_inference(InferenceArgs {
      synthesizer_checkpoint_path: &tts_synthesizer_fs_path,
      text_pipeline_type: text_pipeline_type_or_guess.to_str(),
@@ -283,9 +294,15 @@ async fn process_job_with_cleanup(
  check_file_exists(&output_spectrogram_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
  check_file_exists(&output_metadata_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;

+  info!("All required files exist!");
+
+  info!("Reading metadata file...");
+
  let file_metadata = read_metadata_file(&output_metadata_fs_path)
      .map_err(|e| ProcessSingleJobError::Other(e))?;

+  info!("Deleting metadata file...");
+
  safe_delete_temp_file(&output_metadata_fs_path);

  // ==================== UPLOAD AUDIO TO BUCKET ==================== //
@@ -1,9 +1,10 @@
 use errors::AnyhowResult;
-use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::health_check_state::HealthCheckState;

+use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::health_check_state::HealthCheckState;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_sidecar_client::Tacotron2InferenceSidecarClient;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_sidecar_health_check_client::Tacotron2SidecarHealthCheckClient;
+use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::virtual_lfu_cache::SyncVirtualLfuCache;

 pub struct Tacotron2Dependencies {
  pub inference_command: Tacotron2InferenceCommand,
@@ -26,6 +27,7 @@ pub struct SidecarDeps {
  pub inference_client: Tacotron2InferenceSidecarClient,
  pub health_check_client: Tacotron2SidecarHealthCheckClient,
  pub health_check_state: HealthCheckState,
+  pub virtual_lfu_cache: SyncVirtualLfuCache,
 }

 impl Tacotron2Dependencies {
@@ -53,6 +55,11 @@ impl Tacotron2Dependencies {
    let health_check_client=
        Tacotron2SidecarHealthCheckClient::new(&sidecar_hostname)?;

+    let sidecar_max_synthesizer_models = easyenv::get_env_num(
+      "TTS_SIDECAR_MAX_SYNTHESIZER_MODELS", 3)?;
+
+    let virtual_lfu_cache = SyncVirtualLfuCache::new(sidecar_max_synthesizer_models)?;
+
    Ok(Self {
      inference_command: Tacotron2InferenceCommand::from_env()?,
      waveglow_vocoder_model_filename,
@@ -63,6 +70,7 @@ impl Tacotron2Dependencies {
        inference_client,
        health_check_client,
        health_check_state: HealthCheckState::new(),
+        virtual_lfu_cache,
      }
    })
  }
@@ -5,7 +5,7 @@ use hyper::{Body, Request};
 use hyper::client::Client;
 use log::info;

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
 use mysql_queries::column_types::vocoder_type::VocoderType;

@@ -160,7 +160,16 @@ impl Tacotron2InferenceSidecarClient {

    let client = Client::new();

-    let _maybe_response = client.request(req).await?;
+    let response = client.request(req).await?;
+    let status = response.status();
+
+    info!("Response status: {} ({:?})", status, status.canonical_reason());
+
+    if status.is_server_error() {
+      return Err(anyhow!("sidecar server error: {} {:?}", status, status.canonical_reason()));
+    } else if status.is_client_error() {
+      return Err(anyhow!("sidecar client request error: {} {:?}", status, status.canonical_reason()));
+    }

    Ok(())
  }
@@ -4,7 +4,7 @@ use hyper::client::Client;
 use hyper::Uri;
 use log::info;

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;

 // TODO(bt): As more microservices adopt a common health check protocol,
 //  it may make sense to make this a common library.
@@ -0,0 +1,206 @@
+use std::collections::HashSet;
+use std::sync::{Arc, RwLock};
+
+use anyhow::anyhow;
+use lfu::LFUCache;
+
+use errors::AnyhowResult;
+
+/// This stands in front of the python HTTP sidecar and controls which
+/// models get to remain in memory. The Python sidecar keeps multiple
+/// models around, and Rust dictates which ones to keep and which to
+/// purge.
+pub struct VirtualLfuCache {
+  cache: LFUCache<String, ()>,
+}
+
+/// And the Sync/Send threadsafe + interior mutability version.
+#[derive(Clone)]
+pub struct SyncVirtualLfuCache {
+  cache: Arc::<RwLock<VirtualLfuCache>>
+}
+
+impl VirtualLfuCache {
+
+  pub fn new(capacity: usize) -> AnyhowResult<Self> {
+    let cache = LFUCache::with_capacity(capacity)
+        .map_err(|e| anyhow!("Error creating cache (probably capacity): {:?}", e))?;
+    Ok(Self {
+      cache,
+    })
+  }
+
+  pub fn in_cache(&self, model: &String) -> bool {
+    self.cache.contains(model)
+  }
+
+  /// Returns the evicted entry.
+  pub fn insert_returning_replaced(&mut self, path: &str) -> Option<String> {
+    let initial_keys = self.get_keyset();
+
+    self.cache.set(path.to_string(), ());
+
+    let after_keys = self.get_keyset();
+
+    let difference = initial_keys.difference(&after_keys);
+
+    difference.last()
+        .map(|item| item.to_string())
+  }
+
+  pub fn size(&self) -> usize {
+    self.cache.len()
+  }
+
+  fn get_keyset(&self) -> HashSet<String> {
+    // NB: This only makes sense for our use while we're using small caches.
+    let mut key_set = HashSet::with_capacity(self.size());
+    for (k, _) in self.cache.iter() {
+      key_set.insert(k.to_string());
+    }
+    key_set
+  }
+}
+
+impl SyncVirtualLfuCache {
+
+  pub fn new(capacity: usize) -> AnyhowResult<Self> {
+    let cache = VirtualLfuCache::new(capacity)?;
+    Ok(Self {
+      cache: Arc::new(RwLock::new(cache)),
+    })
+  }
+
+  pub fn in_cache(&self, model: &String) -> AnyhowResult<bool> {
+    let exists = self.cache
+        .read()
+        .map(|cache| cache.in_cache(model))
+        .map_err(|err| anyhow!("mutex error: {:?}", err))?;
+    Ok(exists)
+  }
+
+  /// Returns the evicted entry.
+  pub fn insert_returning_replaced(&self, path: &str) -> AnyhowResult<Option<String>> {
+    let maybe_replaced = self.cache
+        .write()
+        .map(|mut cache| cache.insert_returning_replaced(path))
+        .map_err(|err| anyhow!("mutex error: {:?}", err))?;
+    Ok(maybe_replaced)
+  }
+
+  pub fn size(&self) -> AnyhowResult<usize> {
+    let size = self.cache
+        .read()
+        .map(|cache| cache.size())
+        .map_err(|err| anyhow!("mutex error: {:?}", err))?;
+    Ok(size)
+  }
+
+  fn get_keyset(&self) -> AnyhowResult<HashSet<String>> {
+    let key_set = self.cache
+        .read()
+        .map(|cache| cache.get_keyset())
+        .map_err(|err| anyhow!("mutex error: {:?}", err))?;
+    Ok(key_set)
+  }
+}
+
+#[cfg(test)]
+pub mod tests {
+  use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::virtual_lfu_cache::VirtualLfuCache;
+
+  #[test]
+  fn large_capacity_does_not_shed() {
+    let mut cache = VirtualLfuCache::new(10).unwrap();
+    cache.insert_returning_replaced("1");
+    cache.insert_returning_replaced("2");
+    cache.insert_returning_replaced("3");
+    cache.insert_returning_replaced("4");
+    cache.insert_returning_replaced("5");
+    cache.insert_returning_replaced("6");
+    cache.insert_returning_replaced("7");
+    cache.insert_returning_replaced("8");
+    cache.insert_returning_replaced("9");
+    assert!(cache.insert_returning_replaced("10").is_none()); // Does not remove
+    assert!(cache.insert_returning_replaced("11").is_some()); // Removes
+    assert_eq!(cache.size(), 10);
+  }
+
+  #[test]
+  fn insert_beyond_capacity() {
+    let mut cache = VirtualLfuCache::new(3).unwrap();
+    assert_eq!(cache.size(), 0);
+    cache.insert_returning_replaced("foo");
+    assert_eq!(cache.size(), 1);
+    cache.insert_returning_replaced("bar");
+    assert_eq!(cache.size(), 2);
+    cache.insert_returning_replaced("baz");
+    assert_eq!(cache.size(), 3);
+    cache.insert_returning_replaced("bin");
+    assert_eq!(cache.size(), 3);
+    cache.insert_returning_replaced("bin");
+    assert_eq!(cache.size(), 3);
+    cache.insert_returning_replaced("111111111111111111");
+    assert_eq!(cache.size(), 3);
+  }
+
+  fn repeated_insert_single_key() {
+    let mut cache = VirtualLfuCache::new(3).unwrap();
+    assert_eq!(cache.size(), 0);
+    for _ in 0..10 {
+      cache.insert_returning_replaced("foo");
+    }
+    assert_eq!(cache.size(), 1);
+  }
+
+  #[test]
+  fn returns_first_for_eviction_when_all_used_once() {
+    let mut cache = VirtualLfuCache::new(3).unwrap();
+    cache.insert_returning_replaced("foo");
+    cache.insert_returning_replaced("bar");
+    cache.insert_returning_replaced("baz");
+    let discarded = cache.insert_returning_replaced("bin");
+    assert_eq!(discarded, Some("foo".to_string()));
+
+    assert_eq!(false, cache.in_cache(&"foo".to_string()));
+    assert_eq!(true, cache.in_cache(&"bar".to_string()));
+    assert_eq!(true, cache.in_cache(&"baz".to_string()));
+    assert_eq!(true, cache.in_cache(&"bin".to_string()));
+  }
+
+  #[test]
+  fn retains_frequent_value() {
+    let mut cache = VirtualLfuCache::new(3).unwrap();
+    for _ in 0..10 {
+      cache.insert_returning_replaced("foo");
+    }
+    cache.insert_returning_replaced("bar");
+    cache.insert_returning_replaced("baz");
+    let discarded = cache.insert_returning_replaced("bin");
+    assert_eq!(discarded, Some("bar".to_string()));
+
+    assert_eq!(true, cache.in_cache(&"foo".to_string()));
+    assert_eq!(false, cache.in_cache(&"bar".to_string()));
+    assert_eq!(true, cache.in_cache(&"baz".to_string()));
+    assert_eq!(true, cache.in_cache(&"bin".to_string()));
+  }
+
+  #[test]
+  fn retains_second_frequent_value() {
+    let mut cache = VirtualLfuCache::new(3).unwrap();
+    for _ in 0..10 {
+      cache.insert_returning_replaced("foo");
+    }
+    for _ in 0..2 {
+      cache.insert_returning_replaced("bar");
+    }
+    cache.insert_returning_replaced("baz");
+    let discarded = cache.insert_returning_replaced("bin");
+    assert_eq!(discarded, Some("baz".to_string()));
+
+    assert_eq!(true, cache.in_cache(&"foo".to_string()));
+    assert_eq!(true, cache.in_cache(&"bar".to_string()));
+    assert_eq!(false, cache.in_cache(&"baz".to_string()));
+    assert_eq!(true, cache.in_cache(&"bin".to_string()));
+  }
+}
@@ -177,11 +177,11 @@ pub async fn process_create_voice(
  for downloader in model_dependencies.downloaders.all_downloaders() {
    let result = downloader.download_if_not_on_filesystem(
      &args.job_dependencies.buckets.private_bucket_client,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads
+      &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads
    ).await;
-    if let Err(e) = result {
-      error!("could not download: {:?}", e);
-      return Err(ProcessSingleJobError::from_anyhow_error(e));
+    if let Err(err) = result {
+      error!("could not download: {:?}", err);
+      return Err(err);
    }
  }

@@ -82,12 +82,12 @@ pub async fn process_inference_voice(
  for downloader in model_dependencies.downloaders.all_downloaders() {
    let result = downloader.download_if_not_on_filesystem(
      &args.job_dependencies.buckets.private_bucket_client,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads
+      &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads
    ).await;

-    if let Err(e) = result {
-      error!("could not download: {:?}", e);
-      return Err(ProcessSingleJobError::from_anyhow_error(e));
+    if let Err(err) = result {
+      error!("could not download: {:?}", err);
+      return Err(err);
    }
  }

@@ -2,17 +2,17 @@ use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
 use std::fs::File;
-use std::path::{ Path, PathBuf };
+use std::path::{Path, PathBuf};
 use std::time::Duration;

 use anyhow::anyhow;
 use log::info;
 use once_cell::sync::Lazy;
-use subprocess::{ Popen, PopenConfig, Redirection };
+use subprocess::{Popen, PopenConfig, Redirection};

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
-use subprocess_common::docker_options::{ DockerFilesystemMount, DockerGpu, DockerOptions };
+use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};

 use crate::job::job_loop::command_exit_status::CommandExitStatus;

@@ -7,11 +7,11 @@ use anyhow::anyhow;
 use log::{error, info};
 use tempdir::TempDir;

-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
 use errors::AnyhowResult;
+use filesys::check_file_exists::check_file_exists;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_string::sha256_hash_string;
 use mysql_queries::column_types::vocoder_type::VocoderType;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -22,7 +22,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_types::tts::vits::vits_inference_command::{Device, VitsInferenceArgs};
 use crate::job_dependencies::JobDependencies;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};

 /// Text starting with this will be treated as a test request.
 /// This allows the request to bypass the model cache and query the latest TTS model.
@@ -70,16 +70,17 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result<JobSuccessResul
    // In the future we may need to "repair" broken models.
    let vits_traced_synthesizer_object_path  = args.job_dependencies.buckets.bucket_path_unifier.tts_traced_synthesizer_path(&tts_model.private_bucket_hash);

-    maybe_download_file_from_bucket(
-      "vits traced tts model",
-      &vits_traced_synthesizer_fs_path,
-      &vits_traced_synthesizer_object_path,
-      &args.job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading synthesizer",
-      job.id.0,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "vits traced tts model",
+      final_filesystem_file_path: &vits_traced_synthesizer_fs_path,
+      bucket_object_path: &vits_traced_synthesizer_object_path,
+      bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading synthesizer",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;

    vits_traced_synthesizer_fs_path
  };
@@ -4,12 +4,12 @@ use anyhow::anyhow;
 use log::{error, info};

 use cloud_storage::bucket_client::BucketClient;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use errors::AnyhowResult;
 use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
 use filesys::file_exists::file_exists;
 use filesys::rename_across_devices::rename_across_devices;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;

+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;

 #[derive(Clone)]
@@ -43,7 +43,7 @@ impl PretrainedHubertModel {
    &self,
    bucket_client: &BucketClient,
    scoped_tempdir_creator: &ScopedTempDirCreator,
-  ) -> AnyhowResult<()> {
+  ) -> Result<(), ProcessSingleJobError> {
    if file_exists(&self.filesystem_path) {
      return Ok(());
    }
@@ -1,3 +1,4 @@
+use std::fs::read_to_string;
 use std::path::PathBuf;
 use std::time::Instant;

@@ -6,12 +7,12 @@ use log::{error, info, warn};

 use buckets::public::media_files::original_file::MediaFileBucketPath;
 use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use filesys::check_file_exists::check_file_exists;
 use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
 use filesys::file_size::file_size;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use media::decode_basic_audio_info::decode_basic_audio_file_info;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
@@ -27,7 +28,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::InferenceArgs;
 use crate::job_dependencies::JobDependencies;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};

 const BUCKET_FILE_PREFIX : &str = "fakeyou_rvc_";
 const BUCKET_FILE_EXTENSION : &str = ".wav";
@@ -64,11 +65,11 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu

  model_dependencies.pretrained_hubert_model.download_if_not_on_filesystem(
    &args.job_dependencies.buckets.private_bucket_client,
-    &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads)
+    &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads)
      .await
-      .map_err(|e| {
-        error!("could not download hubert: {:?}", e);
-        ProcessSingleJobError::from_anyhow_error(e)
+      .map_err(|err| {
+        error!("could not download hubert: {:?}", err);
+        err
      })?;

  // ==================== CONFIRM OR DOWNLOAD RVC (v2) MODEL ==================== //
@@ -87,16 +88,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu

    let model_object_path = args.job_dependencies.buckets.bucket_path_unifier.rvc_v2_model_path(&vc_model.private_bucket_hash);

-    maybe_download_file_from_bucket(
-      "rvc (v2) model",
-      &fs_path,
-      &model_object_path,
-      &args.job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading rvc (v2) model",
-      job.id.0,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "rvc (v2) model",
+      final_filesystem_file_path: &fs_path,
+      bucket_object_path: &model_object_path,
+      bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading rvc (v2) model",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;

    fs_path
  };
@@ -113,12 +115,12 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu

    let result = downloader.download_if_not_on_filesystem(
      &args.job_dependencies.buckets.private_bucket_client,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
+      &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
    ).await;

-    if let Err(e) = result {
-      error!("could not download: {:?}", e);
-      return Err(ProcessSingleJobError::from_anyhow_error(e))
+    if let Err(err) = result {
+      error!("could not download: {:?}", err);
+      return Err(err);
    }
  }

@@ -142,16 +144,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu

      let model_index_object_path = args.job_dependencies.buckets.bucket_path_unifier.rvc_v2_model_index_path(&vc_model.private_bucket_hash);

-      maybe_download_file_from_bucket(
-        "rvc (v2) model index",
-        &fs_path,
-        &model_index_object_path,
-        &args.job_dependencies.buckets.private_bucket_client,
-        &mut job_progress_reporter,
-        "downloading rvc (v2) model index",
-        job.id.0,
-        &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-      ).await?;
+      maybe_download_file_from_bucket(MaybeDownloadArgs {
+        name_or_description_of_file: "rvc (v2) model index",
+        final_filesystem_file_path: &fs_path,
+        bucket_object_path: &model_index_object_path,
+        bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+        job_progress_reporter: &mut job_progress_reporter,
+        job_progress_update_description: "downloading rvc (v2) model index",
+        job_id: job.id.0,
+        scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+        maybe_existing_file_minimum_size_required: None,
+      }).await?;

      Some(fs_path)
    };
@@ -185,16 +188,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu

    info!("Downloading media from bucket path: {:?}", &bucket_object_path);

-    maybe_download_file_from_bucket(
-      "media upload (original file)",
-      &original_media_upload_fs_path,
-      &bucket_object_path,
-      &args.job_dependencies.buckets.public_bucket_client,
-      &mut job_progress_reporter,
-      "downloading",
-      job.id.0,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file:  "media upload (original file)",
+      final_filesystem_file_path: &original_media_upload_fs_path,
+      bucket_object_path: &bucket_object_path,
+      bucket_client: &args.job_dependencies.buckets.public_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;

    original_media_upload_fs_path
  };
@@ -217,6 +221,8 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
  //let output_metadata_fs_path = temp_dir.path().join("metadata.json");
  //let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json");

+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+
  info!("Running RVC (v2) VC inference...");

  info!("Expected output audio filename: {:?}", &output_audio_fs_path);
@@ -257,6 +263,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
        rmvpe_path: &model_dependencies.downloaders.rmvpe.filesystem_path,
        input_path: &input_wav_path,
        output_path: &output_audio_fs_path,
+        stderr_output_file: &stderr_output_file,
        maybe_override_f0_method,
        maybe_f0_up_key,
      });
@@ -267,10 +274,26 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu

  if !command_exit_status.is_success() {
    error!("Inference failed: {:?}", command_exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+
    safe_delete_temp_file(&input_wav_path);
    safe_delete_temp_file(&output_audio_fs_path);
    safe_delete_temp_directory(&work_temp_dir);
-    return Err(ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)));
+
+    return Err(error);
  }

  // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
@@ -1,15 +1,16 @@
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Duration;

 use anyhow::anyhow;
 use log::info;
 use once_cell::sync::Lazy;
-use subprocess::{Popen, PopenConfig};
+use subprocess::{Popen, PopenConfig, Redirection};

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
 use mysql_queries::payloads::generic_inference_args::generic_inference_args::FundamentalFrequencyMethodForJob;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
@@ -80,6 +81,8 @@ pub struct InferenceArgs<P: AsRef<Path>, Q: AsRef<Path>> {
  /// --output_audio_filename: output path of wav file.
  pub output_path: P,

+  pub stderr_output_file: P,
+
  /// --f0-method: f0 prediction method
  pub maybe_override_f0_method: Option<FundamentalFrequencyMethodForJob>,

@@ -267,6 +270,11 @@ impl RvcV2InferenceCommand {

    let mut config = PopenConfig::default();

+    info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref());
+
+    let stderr_file = File::create(&args.stderr_output_file)?;
+    config.stderr = Redirection::File(stderr_file);
+
    if !env_vars.is_empty() {
      config.env = Some(env_vars);
    }
@@ -1,3 +1,4 @@
+use std::fs::read_to_string;
 use std::time::Instant;

 use anyhow::anyhow;
@@ -5,12 +6,12 @@ use log::{error, info, warn};

 use buckets::public::media_files::original_file::MediaFileBucketPath;
 use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
-use container_common::filesystem::check_file_exists::check_file_exists;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
+use filesys::check_file_exists::check_file_exists;
 use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
 use filesys::file_size::file_size;
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use media::decode_basic_audio_info::decode_basic_audio_file_info;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
@@ -26,7 +27,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::{Device, InferenceArgs};
 use crate::job_dependencies::JobDependencies;
-use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};

 const BUCKET_FILE_PREFIX : &str = "fakeyou_svc_";
 const BUCKET_FILE_EXTENSION : &str = ".wav";
@@ -76,16 +77,17 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess

    let so_vits_svc_model_object_path  = args.job_dependencies.buckets.bucket_path_unifier.so_vits_svc_model_path(&vc_model.private_bucket_hash);

-    maybe_download_file_from_bucket(
-      "so-vits-svc model",
-      &so_vits_svc_fs_path,
-      &so_vits_svc_model_object_path,
-      &args.job_dependencies.buckets.private_bucket_client,
-      &mut job_progress_reporter,
-      "downloading so-vits-svc model",
-      job.id.0,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "so-vits-svc model",
+      final_filesystem_file_path: &so_vits_svc_fs_path,
+      bucket_object_path: &so_vits_svc_model_object_path,
+      bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading so-vits-svc model",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;

    so_vits_svc_fs_path
  };
@@ -118,16 +120,17 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess

    info!("Downloading media to bucket path: {:?}", &bucket_object_path);

-    maybe_download_file_from_bucket(
-      "media upload (original file)",
-      &original_media_upload_fs_path,
-      &bucket_object_path,
-      &args.job_dependencies.buckets.public_bucket_client,
-      &mut job_progress_reporter,
-      "downloading",
-      job.id.0,
-      &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
-    ).await?;
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+      name_or_description_of_file: "media upload (original file)",
+      final_filesystem_file_path: &original_media_upload_fs_path,
+      bucket_object_path: &bucket_object_path,
+      bucket_client: &args.job_dependencies.buckets.public_bucket_client,
+      job_progress_reporter: &mut job_progress_reporter,
+      job_progress_update_description: "downloading",
+      job_id: job.id.0,
+      scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+      maybe_existing_file_minimum_size_required: None,
+    }).await?;

    original_media_upload_fs_path
  };
@@ -146,6 +149,8 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess

  let output_audio_fs_path = work_temp_dir.path().join("output.wav");

+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+
  info!("Running VC inference...");

  info!("Expected output audio filename: {:?}", &output_audio_fs_path);
@@ -185,6 +190,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
        model_path: &so_vits_svc_fs_path,
        input_path: &input_wav_path,
        output_path: &output_audio_fs_path,
+        stderr_output_file: &stderr_output_file,
        maybe_config_path: None,
        device: Device::Cuda,
        auto_predict_f0,
@@ -197,10 +203,26 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess

  if !command_exit_status.is_success() {
    error!("Inference failed: {:?}", command_exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+
    safe_delete_temp_file(&input_wav_path);
    safe_delete_temp_file(&output_audio_fs_path);
    safe_delete_temp_directory(&work_temp_dir);
-    return Err(ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)));
+
+    return Err(error);
  }

  // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
@@ -1,15 +1,16 @@
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Duration;

 use anyhow::anyhow;
 use log::info;
 use once_cell::sync::Lazy;
-use subprocess::{Popen, PopenConfig};
+use subprocess::{Popen, PopenConfig, Redirection};

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
 use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions};

@@ -90,6 +91,8 @@ pub struct InferenceArgs<P: AsRef<Path>> {
  /// --output_path: output path of converting model to onnx (which we use to test validity)
  pub output_path: P,

+  pub stderr_output_file: P,
+
  /// --auto-predict-f0: turn on or off fundamental frequency auto prediction
  /// This sounds better when left off, but it defaults to *ON* if not specified.
  pub auto_predict_f0: bool,
@@ -366,6 +369,11 @@ impl SoVitsSvcInferenceCommand {

    let mut config = PopenConfig::default();

+    info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref());
+
+    let stderr_file = File::create(&args.stderr_output_file)?;
+    config.stderr = Redirection::File(stderr_file);
+
    if !env_vars.is_empty() {
      config.env = Some(env_vars);
    }
@@ -60,6 +60,10 @@ pub struct JobSystemControls {
  /// This is provided at job start.
  pub scoped_execution: ScopedExecution,

+  // Allow for models not to exist on the filesystem. All jobs will execute when first tried
+  // regardless of whether their models were previously downloaded.
+  pub always_allow_cold_filesystem_cache: bool,
+
  // How many times to skip jobs (on cold filesystem cache) before proceeding with execution.
  pub cold_filesystem_cache_starvation_threshold: u64,

@@ -138,27 +142,27 @@ pub struct ClientDependencies {
 }

 pub struct FileSystemDetails {
-  /// Temporary directory for storing downloads
-  /// (In prod, typically model files from GCS / NFS PVC mount)
-  pub temp_directory_downloads: PathBuf,
-
-  /// Temporary directory for storing work
-  /// (In prod, typically python inference outputs / emptyDir mount)
-  pub temp_directory_work: PathBuf,
-
  /// If the pause file is set and exists on the filesystem,
  /// the job will pause until the file stops existing.
  /// Good for live debugging of production k8s clusters without
  /// redeploying.
  pub maybe_pause_file: Option<PathBuf>,

-  // TODO: Rename
+  /// Temporary directory for storing downloads
  /// Creates temp directories for scratch files
-  pub scoped_temp_dir_creator_for_downloads: ScopedTempDirCreator,
+  pub scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator,
+
+  /// Temporary directory for storing downloads
+  /// (In prod, typically model files from GCS / NFS PVC mount)
+  pub scoped_temp_dir_creator_for_long_lived_downloads: ScopedTempDirCreator,
+
+  /// Temporary directory for storing work
+  /// (In prod, typically python inference outputs / emptyDir mount)
  pub scoped_temp_dir_creator_for_work: ScopedTempDirCreator,

-  // TODO: Rename
  /// Organizes downloaded files
+  /// Directory to store long-term downloads (models)
+  /// (In prod, typically model files from GCS / NFS PVC mount)
  pub semi_persistent_cache: SemiPersistentCacheDir,
 }

@@ -32,8 +32,8 @@ use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
 use config::common_env::CommonEnv;
 use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
-use container_common::anyhow_result::AnyhowResult;
-use container_common::filesystem::check_directory_exists::check_directory_exists;
+use errors::AnyhowResult;
+use filesys::check_directory_exists::check_directory_exists;
 use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
 use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
 use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder;
@@ -119,35 +119,10 @@ async fn main() -> AnyhowResult<()> {
  )?;

  // Where we download models and resources to (typically a shared NFS volume in prod).
-  let temp_directory_downloads = easyenv::get_env_pathbuf_or_default(
-    "TEMP_DIR_DOWNLOADS",
-    PathBuf::from("/tmp/downloads")
-  );

-  // Where we store scratch files for workloads (temporary processing).
-  let temp_directory_work = easyenv::get_env_pathbuf_or_default(
-    "TEMP_DIR_WORK",
-    PathBuf::from("/tmp/work")
-  );
-
-  if !container_environment.server_environment.is_deployed_in_production() {
-    warn!("Creating temporary directories for non-production / development only!");
-    create_dir_all_if_missing(&temp_directory_downloads)?;
-    create_dir_all_if_missing(&temp_directory_work)?;
-  }
-
-  check_directory_exists(&temp_directory_downloads)?;
-  check_directory_exists(&temp_directory_work)?;
-
-  let semi_persistent_cache =
-      SemiPersistentCacheDir::configured_root(&temp_directory_downloads);
-
-  // TODO(bt,2023.05.22): create_all_paths() for some subset of jobs.
-  info!("Creating pod semi-persistent cache dirs...");
-  semi_persistent_cache.create_custom_vocoder_model_path()?;
-  semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?;
-  semi_persistent_cache.create_tts_synthesizer_model_path()?;
-  semi_persistent_cache.create_voice_conversion_model_path()?;
+  let semi_persistent_cache = SemiPersistentCacheDir::configured_root(
+    easyenv::get_env_pathbuf_or_default("SEMI_PERSISTENT_DIR_ROOT",
+                                        PathBuf::from("/tmp/persistent")));

  let db_connection_string =
      easyenv::get_env_string_or_default(
@@ -242,11 +217,16 @@ async fn main() -> AnyhowResult<()> {
      maybe_keepalive_redis_pool,
    },
    fs: FileSystemDetails {
-      temp_directory_downloads: temp_directory_downloads.clone(),
-      temp_directory_work: temp_directory_work.clone(),
      maybe_pause_file: easyenv::get_env_pathbuf_optional("PAUSE_FILE"),
-      scoped_temp_dir_creator_for_downloads: ScopedTempDirCreator::for_directory(&temp_directory_downloads),
-      scoped_temp_dir_creator_for_work: ScopedTempDirCreator::for_directory(&temp_directory_work),
+      scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory(
+        easyenv::get_env_pathbuf_or_default(
+          "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_short_lived"))),
+      scoped_temp_dir_creator_for_long_lived_downloads: ScopedTempDirCreator::for_directory(
+        easyenv::get_env_pathbuf_or_default(
+          "SCOPED_TEMP_DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived"))),
+      scoped_temp_dir_creator_for_work: ScopedTempDirCreator::for_directory(
+        easyenv::get_env_pathbuf_or_default(
+          "SCOPED_TEMP_DIR_WORK", PathBuf::from("/tmp/downloads_long_lived"))),
      semi_persistent_cache,
    },
    buckets: BucketDependencies {
@@ -263,6 +243,7 @@ async fn main() -> AnyhowResult<()> {
    job: JobSystemDependencies {
      system: JobSystemControls {
        scoped_execution,
+        always_allow_cold_filesystem_cache: easyenv::get_env_bool_or_default("ALWAYS_ALLOW_COLD_FILESYSTEM_CACHE", false),
        cold_filesystem_cache_starvation_threshold: easyenv::get_env_num("COLD_FILESYSTEM_CACHE_STARVATION_THRESHOLD", 3)?,
        job_batch_wait_millis: common_env.job_batch_wait_millis,
        job_max_attempts: common_env.job_max_attempts as u16,
@@ -306,6 +287,8 @@ async fn main() -> AnyhowResult<()> {
    },
  };

+  set_up_directories(&job_dependencies)?;
+
  std::thread::spawn(move || {
    let actix_runtime = actix_web::rt::System::new();
    let http_server_handle = launch_http_server(create_server_args);
@@ -322,3 +305,30 @@ async fn main() -> AnyhowResult<()> {
  Ok(())
 }

+fn set_up_directories(job_dependencies: &JobDependencies) -> AnyhowResult<()> {
+  info!("Setting up and checking file system paths for temporary work, ephemeral and persistent downloads, etc...");
+
+  let fs = &job_dependencies.fs;
+
+  if !job_dependencies.job.info.container.server_environment.is_deployed_in_production() {
+    warn!("Creating directories for non-production / development only!");
+
+    create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?;
+    create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?;
+    create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_work.get_base_dir())?;
+  }
+
+  check_directory_exists(fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?;
+  check_directory_exists(fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?;
+  check_directory_exists(fs.scoped_temp_dir_creator_for_work.get_base_dir())?;
+
+  // TODO(bt,2023.05.22): create_all_paths() for some subset of jobs.
+  info!("Creating semi-persistent cache dirs...");
+
+  fs.semi_persistent_cache.create_custom_vocoder_model_path()?;
+  fs.semi_persistent_cache.create_tts_pretrained_vocoder_model_path()?;
+  fs.semi_persistent_cache.create_tts_synthesizer_model_path()?;
+  fs.semi_persistent_cache.create_voice_conversion_model_path()?;
+
+  Ok(())
+}
@@ -7,7 +7,7 @@ use log::{info, warn};
 use once_cell::sync::Lazy;
 use subprocess::{Popen, PopenConfig};

-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;

 use crate::job::job_loop::command_exit_status::CommandExitStatus;
@@ -0,0 +1,55 @@
+use std::io;
+use std::path::{Path, PathBuf};
+
+use errors::AnyhowResult;
+use filesys::check_directory_exists::check_directory_exists;
+use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
+
+/// Utilities for directories serving as file caches
+#[derive(Clone)]
+pub struct FileCacheDir {
+  directory: PathBuf
+}
+
+impl FileCacheDir {
+  pub fn from_env<P: AsRef<Path>>(env_var_name: &str, default_path: P) -> AnyhowResult<Self> {
+    Ok(Self {
+      directory: easyenv::get_env_pathbuf_or_default(env_var_name, default_path),
+    })
+  }
+
+  pub fn create_dir_all_if_missing(&self) -> std::io::Result<()> {
+    create_dir_all_if_missing(&self.directory)
+  }
+
+  pub fn err_unless_exists(&self) -> AnyhowResult<()> {
+    check_directory_exists(&self.directory)?;
+    Ok(())
+  }
+
+  pub fn as_path(&self) -> &Path {
+    &self.directory
+  }
+
+  /// Return the number of files in the directory (non-recursive).
+  /// If there is a concern for a long-running list operation, a `max_count` can be returned.
+  pub fn file_count(&self, maybe_max_count: Option<usize>) -> io::Result<usize> {
+    let mut count = 0;
+    // NB: Two for loops to prevent if-checks in optimized loop case.
+    if let Some(max_count) = maybe_max_count {
+      for entry in std::fs::read_dir(&self.directory)? {
+        let _entry = entry?;
+        if max_count > count {
+          return Ok(max_count);
+        }
+        count += 1;
+      }
+    } else {
+      for entry in std::fs::read_dir(&self.directory)? {
+        let _entry = entry?;
+        count += 1;
+      }
+    }
+    Ok(count)
+  }
+}
@@ -1,9 +1,12 @@
 use std::path::Path;

-use log::{info, warn};
+use log::{error, info, warn};

 use cloud_storage::bucket_client::BucketClient;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::file_size::file_size;
+use filesys::rename_across_devices::{rename_across_devices, RenameError};
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;

 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -11,60 +14,204 @@ use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;

 // TODO(bt, 2022-07-15): Make a concrete type for bucket paths

+pub struct MaybeDownloadArgs<'a> {
+  pub name_or_description_of_file: &'a str,
+  pub final_filesystem_file_path: &'a Path,
+  pub bucket_object_path: &'a Path,
+  pub bucket_client: &'a BucketClient,
+  pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
+  pub job_progress_update_description: &'a str,
+  pub job_id: i64,
+  pub scoped_tempdir_creator: &'a ScopedTempDirCreator,
+  pub maybe_existing_file_minimum_size_required: Option<u64>,
+}
+
 pub async fn maybe_download_file_from_bucket(
-  name_or_description_of_file: &str,
-  file_path: &Path,
-  bucket_object_path: &Path,
-  bucket_client: &BucketClient,
-  job_progress_reporter: &mut Box<dyn JobProgressReporter>,
-  job_progress_update_description: &str,
-  job_id: i64,
-  scoped_tempdir_creator: &ScopedTempDirCreator,
+  args: MaybeDownloadArgs<'_>
 ) -> Result<(), ProcessSingleJobError> {

-  if file_path.exists() {
-    // TODO(bt, 2022-07-15): Check signature of file
-    return Ok(())
+  if args.final_filesystem_file_path.exists() {
+    // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity
+    let mut existing_file_is_valid = true;
+
+    if let Some(existing_file_minimum_size_required) = args.maybe_existing_file_minimum_size_required {
+      // NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file
+      // size, but the Rust system may report a non-zero (but small) number of bytes. This should
+      // later be investigated and the heuristic simplified. The intent here is merely to make sure
+      // we don't consider these okay:
+      //
+      //   -rw-r--r-- 1 root root 55824433 Nov 30 00:12 VM:012rkwsv91zb
+      //   -rw-r--r-- 1 root root 55823149 Nov 30 00:13 VM:79etbx4fdksv
+      //   -rw-r--r-- 1 root root        0 Nov 30 00:52 VM:1dzepsnwzbkc
+      //   -rw-r--r-- 1 root root        0 Nov 30 00:29 VM:7c2df5a36qjb
+      //
+      let size = file_size(args.final_filesystem_file_path)
+          .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+
+      info!("{} exists at path {:?} ; file size = {size}",
+        &args.name_or_description_of_file,&args.final_filesystem_file_path);
+
+      if size < existing_file_minimum_size_required {
+        existing_file_is_valid = false;
+      }
+    }
+
+    if existing_file_is_valid {
+      return Ok(())
+    }
+  } else {
+    warn!("{} does not exist at path: {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path);
  }

-  warn!("{} does not exist at path: {:?}", name_or_description_of_file, &file_path);
-
-  job_progress_reporter.log_status(job_progress_update_description)
+  args.job_progress_reporter.log_status(args.job_progress_update_description)
      .map_err(|e| ProcessSingleJobError::Other(e))?;

  // NB: Download to temp directory to stop concurrent writes and race conditions from other
  // workers writing to a shared volume.
-  let temp_dir = format!("temp_download_{}", job_id);
+  let temp_dir = format!("temp_download_{}", args.job_id);

  // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
-  let temp_dir = scoped_tempdir_creator.new_tempdir(&temp_dir)
+  let temp_dir = args.scoped_tempdir_creator.new_tempdir(&temp_dir)
      .map_err(|e| ProcessSingleJobError::from_io_error(e))?;

  let temp_path = temp_dir.path().join("download.part");

-  info!("Downloading {} from bucket path: {:?}", name_or_description_of_file, &bucket_object_path);
+  info!("Downloading {} from bucket path: {:?}", args.name_or_description_of_file, &args.bucket_object_path);

-  bucket_client.download_file_to_disk(&bucket_object_path, &temp_path)
+  args.bucket_client.download_file_to_disk(&args.bucket_object_path, &temp_path)
      .await
      .map_err(|e| {
        safe_delete_temp_directory(&temp_dir);
        ProcessSingleJobError::Other(e)
      })?;

-  info!("Downloaded {} from bucket!", name_or_description_of_file);
+  info!("Downloaded {} from bucket!", args.name_or_description_of_file);
+
+  let original_size = file_size(&temp_path)
+      .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+
+  info!("File size of {} temp download file {:?} is {original_size}",
+    args.name_or_description_of_file, &temp_path);

  info!("Renaming {} temp file from {:?} to {:?}!",
-    name_or_description_of_file, &temp_path, &file_path);
+    args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path);

-  std::fs::rename(&temp_path, &file_path)
-      .map_err(|e| {
+  rename_across_devices(&temp_path, &args.final_filesystem_file_path)
+      .map_err(|err| {
+        error!("could not rename on disk: {:?}", err);
+        safe_delete_temp_file(&temp_path);
        safe_delete_temp_directory(&temp_dir);
-        ProcessSingleJobError::from_io_error(e)
+        match err {
+          RenameError::StorageFull => ProcessSingleJobError::FilesystemFull,
+          RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err),
+        }
      })?;

-  info!("Finished downloading {} file to {:?}", name_or_description_of_file, &file_path);
+//  info!("Copying {} temp file from {:?} to {:?}!",
+//    args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path);
+//
+//  // NB: We're now seeing a bug where the resultant copied file is 0 bytes
+//  match copy_with_logging(&temp_path, &args.final_filesystem_file_path) {
+//    Err(err) => {
+//      error!("Error Copying {} temp file from {:?} to {:?}! {err}",
+//        args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path);
+//
+//      safe_delete_temp_file(&temp_path);
+//      safe_delete_temp_directory(&temp_dir);
+//
+//      return Err(ProcessSingleJobError::from_anyhow_error(err));
+//    }
+//    Ok(false) => {
+//      error!("Error copying {} temp file from {:?} to {:?}! File size in bytes did not match.",
+//        args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path);
+//
+//      reattempt_copy_if_failed(&args, &temp_dir, &temp_path, original_size)?;
+//    }
+//    Ok(true) => {
+//      // Success case
+//    }
+//  }

+  info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path);
+
+  safe_delete_temp_file(&temp_path);
  safe_delete_temp_directory(&temp_dir);

  Ok(())
-}
+}
+
+// NB: Debugging failures with rename_across_devices() -
+// fn reattempt_copy_if_failed(args: &MaybeDownloadArgs, temp_dir: &TempDir, temp_path: &Path, original_size: u64) -> Result<(), ProcessSingleJobError> {
+//   let copied_size = file_size(&args.final_filesystem_file_path)
+//       .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+//
+//   info!("Final copied size of {} file {:?} is {copied_size} (original size was {original_size})",
+//     args.name_or_description_of_file, &args.final_filesystem_file_path);
+//
+//   if copied_size != 0 {
+//     safe_delete_temp_file(&temp_path);
+//     safe_delete_temp_directory(&temp_dir);
+//
+//     return Ok(());
+//   }
+//
+//   error!("Copied size was 0! Removing destination file and retrying...");
+//
+//   if let Err(err) = std::fs::remove_file(&args.final_filesystem_file_path) {
+//     match err.kind() {
+//       ErrorKind::NotFound => {
+//         // NB: We seem to be seeing this in production.
+//         // Perhaps another pod deleted it in a race condition?
+//         // Fall through case.
+//         warn!("File couldn't be removed: it's already gone.")
+//       },
+//       _ => {
+//         safe_delete_temp_file(&temp_path);
+//         safe_delete_temp_directory(&temp_dir);
+//         return Err(ProcessSingleJobError::from_io_error(err));
+//       }
+//     }
+//   } else {
+//     warn!("File removed.");
+//   }
+//
+//   warn!("Retrying copy...");
+//
+//   rename_across_devices(&temp_path, &args.final_filesystem_file_path)
+//       .map_err(|err| {
+//         error!("could not rename on disk: {:?}", err);
+//         safe_delete_temp_file(&temp_path);
+//         safe_delete_temp_directory(&temp_dir);
+//         match err {
+//           RenameError::StorageFull => ProcessSingleJobError::FilesystemFull,
+//           RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err),
+//         }
+//       })?;
+//
+//   let copied_size = file_size(&args.final_filesystem_file_path)
+//       .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+//
+//   warn!("Last copy attempt. Final copied size of {} file {:?} is {copied_size}.",
+//     args.name_or_description_of_file, &args.final_filesystem_file_path);
+//
+//   Ok(())
+// }
+//
+//
+// pub fn copy_with_logging<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> AnyhowResult<bool> {
+//   let original_size = file_size(&from)?;
+//
+//   let num_bytes_copied = std::fs::copy(&from, &to)?;
+//
+//   let destination_size = file_size(&to)?;
+//
+//   info!(r#"Copied {:?} to {:?}
+//    - Copied bytes: {num_bytes_copied}
+//    - Orig. bytes:  {original_size}
+//    - Dest. bytes:  {destination_size}
+//   "#, from.as_ref(), to.as_ref());
+//
+//   let file_sizes_match = original_size == destination_size;
+//
+//   Ok(file_sizes_match)
+// }
@@ -1,4 +1,5 @@
 pub mod common_commands;
+pub mod file_cache_dir;
 pub mod from_env;
 pub mod maybe_download_file_from_bucket;
 #[macro_use]
@@ -5,12 +5,12 @@ use async_trait::async_trait;
 use log::{error, info};

 use cloud_storage::bucket_client::BucketClient;
-use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
-use errors::AnyhowResult;
 use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
 use filesys::file_exists::file_exists;
-use filesys::rename_across_devices::rename_across_devices;
+use filesys::rename_across_devices::{rename_across_devices, RenameError};
+use filesys::safe_delete_temp_directory::safe_delete_temp_directory;

+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;

 /// Helper utility for downloading pretrained models from GCS.
@@ -31,7 +31,7 @@ pub trait ModelDownloader {
    &self,
    bucket_client: &BucketClient,
    scoped_tempdir_creator: &ScopedTempDirCreator,
-  ) -> AnyhowResult<()> {
+  ) -> Result<(), ProcessSingleJobError> {
    let filesystem_path = self.get_filesystem_path();

    if file_exists(filesystem_path) {
@@ -71,10 +71,13 @@ pub trait ModelDownloader {
    info!("Renaming {} file from {:?} to {:?}!", model_name, &temp_path, filesystem_path);

    rename_across_devices(&temp_path, filesystem_path)
-        .map_err(|e| {
-          error!("could not rename on disk: {:?}", e);
+        .map_err(|err| {
+          error!("could not rename on disk: {:?}", err);
          safe_delete_temp_directory(&temp_dir);
-          anyhow!("couldn't rename disk files: {:?}", e)
+          match err {
+            RenameError::StorageFull => ProcessSingleJobError::FilesystemFull,
+            RenameError::IoError(err) => ProcessSingleJobError::from_io_error(err),
+          }
        })?;

    info!("Finished downloading {} file to {:?}", model_name, filesystem_path);
@@ -15,7 +15,11 @@ impl ScopedTempDirCreator {
    }
  }

+  pub fn get_base_dir(&self) -> &Path {
+    &self.base_dir
+  }
+
  pub fn new_tempdir(&self, name: &str) -> std::io::Result<TempDir> {
    TempDir::new_in(&self.base_dir, name)
  }
-}
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Michael Chung	0033af73ce	Feedback addressed!	2023-12-05 20:26:07 -05:00
Michael Chung	b5fcc3cdfa	Added documentation about the documentation	2023-12-05 19:14:02 -05:00
Michael Chung	e986a44e9d	documentation system open api spec.	2023-12-05 19:00:44 -05:00
Brandon Thomas	28770ef736	fix query issue	2023-12-04 05:00:46 -05:00
Brandon Thomas	894084fba6	Merge pull request #181 from storytold/brandon-queue-controls Kill inference jobs (mod endpoint)	2023-12-04 03:54:05 -05:00
Brandon Thomas	4bf1a25fbb	Kill inference jobs (mod endpoint)	2023-12-04 03:46:29 -05:00
Brandon Thomas	b0a9342194	updated queue stats	2023-12-01 07:09:23 -05:00
Brandon Thomas	3cc1ea6935	capture stderr for svc and rvc	2023-12-01 01:36:33 -05:00
Brandon Thomas	b74cf7853d	fix web break	2023-11-30 22:46:35 -05:00
Brandon Thomas	3cdce86162	warn on slow batch queries	2023-11-30 17:25:23 -05:00
Brandon Thomas	b0a87e83a0	log batch index	2023-11-30 17:19:23 -05:00
Brandon Thomas	979c234ed2	cleanup logging	2023-11-30 16:27:25 -05:00
Brandon Thomas	4837106d37	distinguish logs	2023-11-30 16:10:01 -05:00
Brandon Thomas	cfd92e814d	Merge pull request #166 from storytold/dependabot/cargo/env_logger-0.10.1 Bump env_logger from 0.7.1 to 0.10.1	2023-11-30 16:02:23 -05:00
Brandon Thomas	6d27214a14	codegen	2023-11-30 15:50:05 -05:00
Brandon Thomas	b628fb0a11	premium TTS duration	2023-11-30 15:21:07 -05:00
dependabot[bot]	456b9e1ce6	Bump env_logger from 0.7.1 to 0.10.1 Bumps [env_logger](https://github.com/rust-cli/env_logger) from 0.7.1 to 0.10.1. - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.7.1...v0.10.1) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-30 09:35:18 +00:00
Brandon Thomas	6f7b6da595	Merge pull request #167 from storytold/dependabot/cargo/rand-0.8.5 Bump rand from 0.7.3 to 0.8.5	2023-11-30 04:33:54 -05:00
Brandon Thomas	18ad28135f	safe delete	2023-11-30 02:36:33 -05:00
Brandon Thomas	ff4a3115d2	clear up more directories when file system is full	2023-11-30 02:34:33 -05:00
dependabot[bot]	ccae943037	Bump rand from 0.7.3 to 0.8.5 Bumps [rand](https://github.com/rust-random/rand) from 0.7.3 to 0.8.5. - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.7.3...0.8.5) --- updated-dependencies: - dependency-name: rand dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-30 07:32:36 +00:00
Brandon Thomas	12ef105ced	fix errors	2023-11-30 02:27:08 -05:00
Brandon Thomas	b27070e23e	get rid of more of container_common	2023-11-30 02:25:16 -05:00
Brandon Thomas	8e45a4ea45	delete mod	2023-11-30 02:05:15 -05:00
Brandon Thomas	a669ac7f03	move remaining container_common::files -> filesys crate	2023-11-30 02:04:41 -05:00
Brandon Thomas	b5ed9acdd4	move `check_directory_exists` to filesys crate	2023-11-30 01:56:55 -05:00
Brandon Thomas	ff2aaa9f3d	move `check_file_exists` to filesys crate	2023-11-30 01:53:42 -05:00
Brandon Thomas	f93421eadc	keep IoError around; better casting	2023-11-30 01:35:48 -05:00
Brandon Thomas	b82ffcce86	much more StorageFull error !	2023-11-30 01:26:03 -05:00
Brandon Thomas	e2c5839d01	surface storage full errors	2023-11-30 01:05:26 -05:00
Brandon Thomas	896e8591ba	cleanup	2023-11-29 23:10:15 -05:00
Brandon Thomas	19dcfdc629	cleanup	2023-11-29 22:51:12 -05:00
Brandon Thomas	cec7bce450	try copy instead	2023-11-29 22:49:00 -05:00
Brandon Thomas	6cc3062da8	don't fail if file is gone (not sure why this happens)	2023-11-29 22:01:53 -05:00
Brandon Thomas	efdd03bbbd	logging	2023-11-29 21:50:30 -05:00
Brandon Thomas	c207356ac5	cleanup retry	2023-11-29 21:44:20 -05:00
Brandon Thomas	3243cc51b7	try to delete and re-copy	2023-11-29 21:36:47 -05:00
Brandon Thomas	ee4af2976f	flag to permit cold filesystem cache	2023-11-29 21:26:03 -05:00
Brandon Thomas	4fb2a38c68	report temp file size	2023-11-29 21:10:01 -05:00
Brandon Thomas	a549ceb609	check vocoder and synthesizer sizes	2023-11-29 20:53:00 -05:00
Brandon Thomas	003f0cdeea	struct args for downloader (safer)	2023-11-29 20:46:33 -05:00
Brandon Thomas	9e0772141a	download file size check and logging	2023-11-29 20:25:37 -05:00
Brandon Thomas	542b79dd90	log http errors	2023-11-29 19:49:52 -05:00
Brandon Thomas	363f8cbc96	debug log statements, FileCacheDir (not yet used)	2023-11-29 19:28:32 -05:00
Brandon Thomas	f2e75f8637	rename files across fs devices	2023-11-29 18:46:00 -05:00
Brandon Thomas	f48cbb7978	fix	2023-11-29 18:16:45 -05:00
Brandon Thomas	edd5cd6156	Merge pull request #179 from storytold/brandon-fix-job-pathing Brandon fix job pathing	2023-11-29 17:49:39 -05:00
Brandon Thomas	37c016cecb	"semi persistent dir" needs cleanup, this is confusing	2023-11-29 17:48:29 -05:00
Brandon Thomas	1a3e111dd6	Fix job pathing for outage and to make easier to manage	2023-11-29 17:30:03 -05:00
Brandon Thomas	0c456d957c	fix build errors	2023-11-29 15:46:43 -05:00
Brandon Thomas	85ecce7b38	comments	2023-11-29 03:07:07 -05:00
Brandon Thomas	2f1d682438	Merge pull request #176 from storytold/nov-29-mithoo fail model deleted inference jobs	2023-11-29 02:44:38 -05:00
Brandon Thomas	c6fbfaa6ba	fix test failures and nits	2023-11-29 02:43:31 -05:00
Brandon Thomas	038401bff4	Merge pull request #177 from storytold/brandon-revert-sha2-bump Revert "Bump sha2 from 0.9.9 to 0.10.8"	2023-11-29 02:29:16 -05:00
Brandon Thomas	e1c7231d96	Revert "Bump sha2 from 0.9.9 to 0.10.8" This reverts commit `951afc2e5d`.	2023-11-29 02:28:51 -05:00
kasisnu	1e11fdb35d	fail model deleted inference jobs	2023-11-29 12:43:49 +05:30
Brandon Thomas	74c477b045	Merge pull request #169 from storytold/dependabot/cargo/sha2-0.10.8 Bump sha2 from 0.9.9 to 0.10.8	2023-11-28 21:37:50 -05:00
Brandon Thomas	042fa1574d	Merge pull request #168 from storytold/dependabot/cargo/time-0.3.30 Bump time from 0.2.27 to 0.3.30	2023-11-28 21:37:26 -05:00
Brandon Thomas	4387e6326f	LFU cache in TT2 inference-job	2023-11-28 17:47:46 -05:00
Brandon Thomas	3ad2b744b7	test comment out Dockerfile /etc/ssl copy	2023-11-28 15:42:24 -05:00
Brandon Thomas	8bf85b637b	notes on dependencies	2023-11-28 15:40:26 -05:00
Brandon Thomas	50c3b5ed65	codegen db	2023-11-28 15:23:43 -05:00
Brandon Thomas	df15395bf7	fix build error	2023-11-28 15:08:39 -05:00
Brandon Thomas	5666d58da0	fix SSL problem?	2023-11-28 14:54:53 -05:00
dependabot[bot]	0a85e9c581	Bump time from 0.2.27 to 0.3.30 Bumps [time](https://github.com/time-rs/time) from 0.2.27 to 0.3.30. - [Release notes](https://github.com/time-rs/time/releases) - [Changelog](https://github.com/time-rs/time/blob/main/CHANGELOG.md) - [Commits](https://github.com/time-rs/time/compare/v0.2.27...v0.3.30) --- updated-dependencies: - dependency-name: time dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-28 16:45:21 +00:00
ArEnSc	bd0ce25d94	Merge pull request #151 from storytold/feature/weights_queries Feature/weights queries	2023-11-28 11:43:00 -05:00
dependabot[bot]	951afc2e5d	Bump sha2 from 0.9.9 to 0.10.8 Bumps [sha2](https://github.com/RustCrypto/hashes) from 0.9.9 to 0.10.8. - [Commits](https://github.com/RustCrypto/hashes/compare/sha2-v0.9.9...sha2-v0.10.8) --- updated-dependencies: - dependency-name: sha2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-11-27 22:32:08 +00:00