add model downloading

crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs

@@ -26,8 +26,8 @@ pub enum InferenceCategory {
   #[serde(rename = "voice_conversion")]
   VoiceConversion,
 
-  #[serde(rename = "rerender_a_video")]
-  RerenderAVideo,
+  #[serde(rename = "video_filter")]
+  VideoFilter,
 }
 
 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -41,7 +41,7 @@ impl InferenceCategory {
       Self::LipsyncAnimation => "lipsync_animation",
       Self::TextToSpeech => "text_to_speech",
       Self::VoiceConversion => "voice_conversion",
-      Self::RerenderAVideo => "rerender_a_video",
+      Self::VideoFilter => "video_filter",
     }
   }
 
@@ -50,7 +50,7 @@ impl InferenceCategory {
       "lipsync_animation" => Ok(Self::LipsyncAnimation),
       "text_to_speech" => Ok(Self::TextToSpeech),
       "voice_conversion" => Ok(Self::VoiceConversion),
-      "rerender_a_video" => Ok(Self::RerenderAVideo),
+      "video_filter" => Ok(Self::VideoFilter),
       _ => Err(format!("invalid value: {:?}", value)),
     }
   }
@@ -62,7 +62,7 @@ impl InferenceCategory {
       Self::LipsyncAnimation,
       Self::TextToSpeech,
       Self::VoiceConversion,
-      Self::RerenderAVideo,
+      Self::VideoFilter,
     ])
   }
 }
@@ -77,7 +77,7 @@ mod tests {
     assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation");
     assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech");
     assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion");
-    assert_serialization(InferenceCategory::RerenderAVideo, "rerender_a_video");
+    assert_serialization(InferenceCategory::VideoFilter, "video_filter");
   }
 
   #[test]
@@ -85,7 +85,7 @@ mod tests {
     assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation");
     assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech");
     assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion");
-    assert_eq!(InferenceCategory::RerenderAVideo.to_str(), "rerender_a_video");
+    assert_eq!(InferenceCategory::VideoFilter.to_str(), "video_filter");
   }
 
   #[test]
@@ -93,7 +93,7 @@ mod tests {
     assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation);
     assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech);
     assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion);
-    assert_eq!(InferenceCategory::from_str("rerender_a_video").unwrap(), InferenceCategory::RerenderAVideo);
+    assert_eq!(InferenceCategory::from_str("video_filter").unwrap(), InferenceCategory::VideoFilter);
   }
 
   #[test]
@@ -104,7 +104,7 @@ mod tests {
     assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation));
     assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech));
     assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion));
-    assert_eq!(variants.pop_first(), Some(InferenceCategory::RerenderAVideo));
+    assert_eq!(variants.pop_first(), Some(InferenceCategory::VideoFilter));
     assert_eq!(variants.pop_first(), None);
 
     // Generated check

crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs

@@ -21,9 +21,9 @@ pub enum IdCategory {
   #[serde(rename = "lipsync_animation")]
   LipsyncAnimationResult,
 
-  // Results from rerender a video
-  #[serde(rename = "rerender")]
-  RerenderResult,
+  // Results from video filters
+  #[serde(rename = "video_filter")]
+  VideoFilterResult,
 
   /// Results from tacotron2
   /// Applies for RVC and SVC
@@ -66,7 +66,7 @@ impl IdCategory {
       Self::ZeroShotVoiceDataset => "zs_dataset",
       Self::ZeroShotVoiceEmbedding => "zs_voice",
       Self::ZeroShotTtsResult => "zs_tts_result",
-      Self::RerenderResult => "rerender",
+      Self::VideoFilterResult => "video_filter",
       Self::ModelWeights => "model_weights",
     }
   }
@@ -80,7 +80,7 @@ impl IdCategory {
       "zs_dataset" => Ok(Self::ZeroShotVoiceDataset),
       "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding),
       "zs_tts_result" => Ok(Self::ZeroShotTtsResult),
-      "rerender" => Ok(Self::RerenderResult),
+      "video_filter" => Ok(Self::VideoFilterResult),
       "model_weights" => Ok(Self::ModelWeights),
       _ => Err(format!("invalid value: {:?}", value)),
     }
@@ -92,7 +92,7 @@ impl IdCategory {
     BTreeSet::from([
       Self::MediaFile,
       Self::LipsyncAnimationResult,
-      Self::RerenderResult,
+      Self::VideoFilterResult,
       Self::TtsResult,
       Self::VoiceConversionResult,
       Self::ZeroShotTtsResult,
@@ -115,7 +115,7 @@ mod tests {
   fn test_serialization() {
     assert_serialization(IdCategory::MediaFile, "media_file");
     assert_serialization(IdCategory::LipsyncAnimationResult, "lipsync_animation");
-    assert_serialization(IdCategory::RerenderResult, "rerender");
+    assert_serialization(IdCategory::VideoFilterResult, "video_filter");
     assert_serialization(IdCategory::TtsResult, "tts_result");
     assert_serialization(IdCategory::VoiceConversionResult, "voice_conversion");
     assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset");
@@ -128,7 +128,7 @@ mod tests {
     fn to_str() {
       assert_eq!(IdCategory::MediaFile.to_str(), "media_file");
       assert_eq!(IdCategory::LipsyncAnimationResult.to_str(), "lipsync_animation");
-      assert_eq!(IdCategory::RerenderResult.to_str(), "rerender");
+      assert_eq!(IdCategory::VideoFilterResult.to_str(), "video_filter");
       assert_eq!(IdCategory::TtsResult.to_str(), "tts_result");
       assert_eq!(IdCategory::VoiceConversionResult.to_str(), "voice_conversion");
       assert_eq!(IdCategory::ZeroShotVoiceDataset.to_str(), "zs_dataset");
@@ -141,7 +141,7 @@ mod tests {
     fn from_str() {
       assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile);
       assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimationResult);
-      assert_eq!(IdCategory::from_str("rerender").unwrap(), IdCategory::RerenderResult);
+      assert_eq!(IdCategory::from_str("video_filter").unwrap(), IdCategory::VideoFilterResult);
       assert_eq!(IdCategory::from_str("tts_result").unwrap(), IdCategory::TtsResult);
       assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversionResult);
       assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZeroShotVoiceDataset);
@@ -157,7 +157,7 @@ mod tests {
       assert_eq!(variants.len(), 9);
       assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile));
       assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult));
-      assert_eq!(variants.pop_first(), Some(IdCategory::RerenderResult));
+      assert_eq!(variants.pop_first(), Some(IdCategory::VideoFilterResult));
       assert_eq!(variants.pop_first(), Some(IdCategory::TtsResult));
       assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult));
       assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotTtsResult));

crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs

@@ -39,9 +39,9 @@ pub enum MediaFileOriginProductCategory {
   #[serde(rename = "zs_voice")]
   ZeroShotVoice,
 
-  // Media files for Rerender
-  #[serde(rename = "rerender")]
-  Rerender,
+  // Media files for video filters
+  #[serde(rename = "video_filter")]
+  VideoFilter,
 }
 
 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -58,7 +58,7 @@ impl MediaFileOriginProductCategory {
       Self::Unknown => "unknown",
       Self::VoiceConversion => "voice_conversion",
       Self::ZeroShotVoice => "zs_voice",
-      Self::Rerender => "rerender",
+      Self::VideoFilter => "video_filter",
     }
   }
 
@@ -69,7 +69,7 @@ impl MediaFileOriginProductCategory {
       "unknown" => Ok(Self::Unknown),
       "voice_conversion" => Ok(Self::VoiceConversion),
       "zs_voice" => Ok(Self::ZeroShotVoice),
-      "rerender" => Ok(Self::Rerender),
+      "video_filter" => Ok(Self::VideoFilter),
       _ => Err(format!("invalid value: {:?}", value)),
     }
   }
@@ -80,7 +80,7 @@ impl MediaFileOriginProductCategory {
     BTreeSet::from([
       Self::FaceAnimator,
       Self::TextToSpeech,
-      Self::Rerender,
+      Self::VideoFilter,
       Self::Unknown,
       Self::VoiceConversion,
       Self::ZeroShotVoice,
@@ -103,7 +103,7 @@ mod tests {
       assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown");
       assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion");
       assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice");
-      assert_serialization(MediaFileOriginProductCategory::Rerender, "rerender")
+      assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter")
     }
 
     #[test]
@@ -113,7 +113,7 @@ mod tests {
       assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown");
       assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion");
       assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice");
-      assert_eq!(MediaFileOriginProductCategory::Rerender.to_str(), "rerender");
+      assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter");
     }
 
     #[test]
@@ -123,7 +123,7 @@ mod tests {
       assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown);
       assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion);
       assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice);
-      assert_eq!(MediaFileOriginProductCategory::from_str("rerender").unwrap(), MediaFileOriginProductCategory::Rerender);
+      assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter);
     }
 
     #[test]
@@ -135,7 +135,7 @@ mod tests {
       assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown));
       assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion));
       assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice));
-      assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Rerender));
+      assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter));
       assert_eq!(variants.pop_first(), None);
     }
   }

crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs

@@ -37,9 +37,9 @@ pub enum InferenceCategoryAbbreviated {
   #[serde(alias = "voice_conversion")]
   VoiceConversion,
 
-  #[serde(rename = "rr")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
-  #[serde(alias = "rerender_a_video")]
-  RerenderAVideo,
+  #[serde(rename = "vf")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(alias = "video_filter")]
+  VideoFilter,
 }
 
 #[derive(Serialize, Deserialize, Clone, Copy, Debug)]
@@ -112,7 +112,7 @@ impl InferenceCategoryAbbreviated {
       InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation,
       InferenceCategory::TextToSpeech => Self::TextToSpeech,
       InferenceCategory::VoiceConversion => Self::VoiceConversion,
-      InferenceCategory::RerenderAVideo => Self::RerenderAVideo,
+      InferenceCategory::VideoFilter => Self::VideoFilter,
     }
   }
 
@@ -121,7 +121,7 @@ impl InferenceCategoryAbbreviated {
       Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation,
       Self::TextToSpeech => InferenceCategory::TextToSpeech,
       Self::VoiceConversion => InferenceCategory::VoiceConversion,
-      Self::RerenderAVideo => InferenceCategory::RerenderAVideo,
+      Self::VideoFilter => InferenceCategory::VideoFilter,
     }
   }
 }

crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs

@@ -1,3 +1,5 @@
+use tokens::tokens::model_weights::ModelWeightToken;
+
 /// Video sources can be one of several:
 ///  - F: media_files (todo)
 ///  - U: media_uploads (legacy)
@@ -32,5 +34,25 @@ pub struct RerenderArgs {
 
     #[serde(rename = "sd")]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub maybe_sd_model: Option<String>,
+    pub maybe_sd_model_token: Option<ModelWeightToken>,
+
+    #[serde(rename = "lora")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub maybe_lora_model_token: Option<ModelWeightToken>,
+
+    #[serde(rename = "p")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub maybe_prompt: Option<String>,
+
+    #[serde(rename = "ap")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub maybe_a_prompt: Option<String>,
+
+    #[serde(rename = "np")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub maybe_n_prompt: Option<String>,
+
+    #[serde(rename = "se")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub maybe_seed: Option<i32>,
 }

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -178,7 +178,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
     InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
     InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
     InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
-    InferenceCategory::RerenderAVideo => Some("rerender animation"),
+    InferenceCategory::VideoFilter => Some("Video Filter"),
   };
 
   // NB: A bit of a hack. We store TTS results with a full path.
@@ -187,7 +187,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
     InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
     InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
-    InferenceCategory::RerenderAVideo => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
+    InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
   };
 
   // NB: We've moved voice conversion out of their own table and into media_files

crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs

@@ -55,7 +55,7 @@ pub async fn insert_media_file_from_rerender(
 
         let next_rerender_id = transactional_increment_generic_synthetic_id(
             &user_token,
-            IdCategory::RerenderResult,
+            IdCategory::VideoFilterResult,
             &mut transaction
         ).await?;
 
@@ -67,7 +67,7 @@ pub async fn insert_media_file_from_rerender(
     let creator_set_visibility = args.job.creator_set_visibility.clone();
 
     const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference;
-    const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Rerender;
+    const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::VideoFilter;
     const ORIGIN_MODEL_TYPE : MediaFileOriginModelType = MediaFileOriginModelType::Rerender;
     const MEDIA_TYPE : MediaFileType = MediaFileType::Video;
 

crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs

@@ -33,7 +33,7 @@ pub struct RetrivedModelWeight {
     pub file_size_bytes: i32,
     pub file_checksum_sha2: String,
     pub private_bucket_hash: String,
-    pub maybe_private_bucket_prefix: Option<String>,
+    pub maybe_private_bucket_prefix: Option<String>, // TODO crossproduct rename these to public thx ^-^
     pub maybe_private_bucket_extension: Option<String>,
     pub cached_user_ratings_negative_count: u32,
     pub cached_user_ratings_positive_count: u32,

crates/service/job/inference_job/src/job/job_loop/process_single_job.rs

@@ -16,7 +16,7 @@ use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSucce
 use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
 use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job;
 use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job;
-use crate::job::job_types::videofilter::process_single_vf_job::process_single_rr_job;
+use crate::job::job_types::videofilter::process_single_vf_job::process_single_vf_job;
 use crate::job_dependencies::JobDependencies;
 
 pub async fn process_single_job(
@@ -183,8 +183,8 @@ async fn do_process_single_job(
     InferenceCategory::VoiceConversion => {
       process_single_vc_job(job_dependencies, job).await?
     }
-    InferenceCategory::RerenderAVideo => {
-      process_single_rr_job(job_dependencies, job).await?
+    InferenceCategory::VideoFilter => {
+      process_single_vf_job(job_dependencies, job).await?
     }
   };
 

crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs

@@ -1,11 +1,4 @@
-use anyhow::anyhow;
-use log::info;
-
-use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr;
-use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::media_files::get_media_file::get_media_file;
-use tokens::tokens::media_files::MediaFileToken;
 
 use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -13,59 +6,7 @@ use crate::job::job_types::videofilter::rerender_a_video;
 use crate::job::job_types::videofilter::rerender_a_video::process_job::RerenderProcessJobArgs;
 use crate::job_dependencies::JobDependencies;
 
-pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
-    // let maybe_inference_args = job.maybe_inference_args.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Inference args not found")))?;
-    //
-    // let maybe_rerender_args = maybe_inference_args.args.as_ref().map(|args| match args {
-    //     Rr(args) => Some(args),
-    //     _ => None,
-    // }).flatten();
-    //
-    // let rerender_args = match maybe_rerender_args {
-    //     None => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))),
-    //     Some(args) => args,
-    // };
-    //
-    // let videofilter_source = rerender_args.maybe_video_source.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Video source not found")))?;
-    //
-    // let media_file_token = match videofilter_source {
-    //     VideofilterVideoSource::F(token) => token,
-    //     _ => return Err(ProcessSingleJobError::Other(anyhow!("Video source not found"))),
-    // };
-    //
-    // let media_file_token = MediaFileToken::new_from_str(media_file_token);
-    //
-    // info!("media_file_token: {:?}", media_file_token);
-    //
-    //
-    // let media_file = match media_file_result {
-    //     Ok(Some(result)) => result,
-    //     Ok(None) => {
-    //         return Err(ProcessSingleJobError::Other(anyhow!("Media file not found")));
-    //     }
-    //     Err(e) => {
-    //         return Err(ProcessSingleJobError::Other(anyhow!("Media file not found")));
-    //     }
-    // };
-
-    // info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes);
-    // info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis);
-    // info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0));
-
-
-
-    // let job_success_result = match vc_model.model_type {
-    //     VoiceConversionModelType::RvcV2 => {
-    //         rvc_v2::process_job::process_job(RvcV2ProcessJobArgs {
-    //             job_dependencies,
-    //             job,
-    //             vc_model: &vc_model,
-    //             media_upload_token: &media_upload_token,
-    //             media_upload: &media_upload,
-    //         }).await?
-    //     }
-    // };
-
+pub async fn process_single_vf_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
     let job_success_result = rerender_a_video::process_job::process_job(
         RerenderProcessJobArgs {
             job_dependencies,

crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs

@@ -0,0 +1,64 @@
+use std::path::PathBuf;
+
+use anyhow::anyhow;
+use tempdir::TempDir;
+
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
+use cloud_storage::bucket_client::BucketClient;
+use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::model_weights::get_weight::RetrivedModelWeight;
+
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
+use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
+
+pub struct ModelFile {
+    pub filesystem_path: PathBuf,
+}
+
+pub async fn download_model_file(
+    model_record: &Option<RetrivedModelWeight>,
+    public_bucket_client: &BucketClient,
+    job_progress_reporter: &mut Box<dyn JobProgressReporter>,
+    job: &AvailableInferenceJob,
+    temp_dir_creator: &ScopedTempDirCreator,
+    work_temp_dir: &TempDir,
+) -> Result<ModelFile, ProcessSingleJobError> {
+
+    let bucket_object_path;
+    let model_filename;
+    match model_record {
+        Some(model_record) => {
+            let model_bucket_path =
+                MediaFileBucketPath::from_object_hash(
+                    &model_record.private_bucket_hash,
+                    model_record.maybe_private_bucket_prefix.as_deref(),
+                    model_record.maybe_private_bucket_extension.as_deref(),
+                );
+            model_filename = model_record.private_bucket_hash.clone() + ".".to_owned() + model_record.maybe_private_bucket_extension.as_deref().unwrap_or("bin");
+            bucket_object_path = model_bucket_path.to_full_object_pathbuf();
+        }
+        None => {
+            return Err(ProcessSingleJobError::from_anyhow_error(
+                anyhow!("could not find model file")))
+        }
+    };
+    let downloaded_filesystem_path = work_temp_dir.path().join(model_filename);
+
+    maybe_download_file_from_bucket(MaybeDownloadArgs {
+        name_or_description_of_file: "model file",
+        final_filesystem_file_path: &downloaded_filesystem_path,
+        bucket_object_path: &bucket_object_path,
+        bucket_client: public_bucket_client,
+        job_progress_reporter,
+        job_progress_update_description: "downloading",
+        job_id: job.id.0,
+        scoped_tempdir_creator: &temp_dir_creator,
+        maybe_existing_file_minimum_size_required: None,
+    }).await?;
+
+    Ok(ModelFile {
+        filesystem_path: downloaded_filesystem_path,
+    })
+}

crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs

@@ -4,3 +4,4 @@ mod download_video_file;
 mod rerender_inference_command;
 pub(crate) mod rerender_dependencies;
 mod categorize_error;
+mod download_model_file;

crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs

@@ -16,9 +16,11 @@ use mimetypes::mimetype_for_file::get_mimetype_for_file;
 use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs};
+use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
 
 use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::videofilter::rerender_a_video::download_model_file::download_model_file;
 use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file;
 use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs;
 use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job;
@@ -122,22 +124,69 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result<JobSuccessR
         _ => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))),
     };
 
-    let sd_model = rr_args.maybe_sd_model.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?;
+
+    // =================== LOAD MODELS AND LORAS ==================== //
+
+    let sd_model_token = rr_args.maybe_sd_model_token.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?;
+
+    let retrieved_sd_record =  get_weight_by_token(
+        sd_model_token,
+        false,
+        &deps.db.mysql_pool
+    ).await?;
+
+    let model_path = download_model_file(
+        &retrieved_sd_record,
+        &args.job_dependencies.buckets.public_bucket_client,
+        &mut job_progress_reporter,
+        job,
+        &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+        &work_temp_dir).await?;
+
+    info!("Downloaded model file: {:?}", model_path.filesystem_path);
+    let sd_model = model_path.filesystem_path.to_str().unwrap().to_string();
+
+    let lora_path = match rr_args.maybe_lora_model_token.clone() {
+        Some(lora_model_token) => {
+            let retrieved_lora_record = get_weight_by_token(
+                &lora_model_token,
+                false,
+                &deps.db.mysql_pool
+            ).await?;
+
+            let lora_path = download_model_file(
+                &retrieved_lora_record,
+                &args.job_dependencies.buckets.public_bucket_client,
+                &mut job_progress_reporter,
+                job,
+                &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+                &work_temp_dir).await?;
+
+            info!("Downloaded lora file: {:?}", lora_path.filesystem_path);
+
+            Some(lora_path.filesystem_path)
+        }
+        None => None
+    };
+    let sd_lora = match lora_path {
+        Some(lora_path) => Some(lora_path.to_str().unwrap().to_string()),
+        None => None
+    };
 
     // ============== CREATE AND WRITE CONFIG TO DISK  ============== //
 
     // create seed if not exists
-    // let seed = rr_args.seed.unwrap_or_else(|| rand::random::<i64>());
-    let seed = rand::random::<i32>();
+    let seed = rr_args.maybe_seed.unwrap_or_else(|| rand::random::<i32>());
 
     let config = RerenderConfig {
         input: Some(usable_video_path.to_str().unwrap().to_string()),
         output: Some(output_video_fs_path.to_str().unwrap().to_string()),
         work_dir: Some(work_temp_dir.path().to_str().unwrap().to_string()),
         sd_model: Some(sd_model.to_string()),
-        prompt: Some("1girl, anime style, red hair".to_string()),
-        a_prompt: Some("1girl, 3d, realistic, read hair".to_string()),
-        n_prompt: Some("Bad quality, low quality, photo, artifacts, watermark, signature, faceless".to_string()),
+        lora_path: sd_lora,
+        prompt: rr_args.maybe_prompt.clone(),
+        a_prompt: rr_args.maybe_a_prompt.clone(),
+        n_prompt: rr_args.maybe_n_prompt.clone(),
         seed: Some(seed as i64),
         ..RerenderConfig::new()
     };
@@ -170,14 +219,6 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result<JobSuccessR
 
         if let Ok(contents) = read_to_string(&stderr_output_file) {
             warn!("Captured stderr output: {}", contents);
-
-            // match categorize_error(&contents)  {
-            //     Some(ProcessSingleJobError::FaceDetectionFailure) => {
-            //         warn!("Face not detected in source image");
-            //         error = ProcessSingleJobError::FaceDetectionFailure;
-            //     }
-            //     _ => {}
-            // }
         }
 
         safe_delete_temp_file(&video_path.filesystem_path);
@@ -234,7 +275,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result<JobSuccessR
 
     let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
 
-    info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf);
+    info!("Video destination bucket path: {:?}", &result_bucket_object_pathbuf);
 
     info!("Uploading media ...");
 

crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs

@@ -3,12 +3,18 @@ use anyhow::anyhow;
 use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
 use mysql_queries::payloads::generic_inference_args::videofilter_payload::{VideofilterVideoSource};
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use tokens::tokens::model_weights::ModelWeightToken;
 
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 
 pub struct JobArgs<'a> {
   pub video_source: &'a VideofilterVideoSource,
-  pub sd_model: &'a str,
+  pub sd_model_token: &'a ModelWeightToken,
+  pub lora_model_token: &'a Option<ModelWeightToken>,
+  pub prompt: &'a str,
+  pub a_prompt: &'a str,
+  pub n_prompt: &'a str,
+  pub seed: Option<i32>,
 }
 
 pub fn validate_job(job: &AvailableInferenceJob) -> Result<JobArgs, ProcessSingleJobError> {
@@ -23,7 +29,7 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result<JobArgs, ProcessSingl
       .flatten();
 
   match inference_category {
-    Some(InferenceCategoryAbbreviated::RerenderAVideo) => {}, // Valid
+    Some(InferenceCategoryAbbreviated::VideoFilter) => {}, // Valid
     Some(category) => {
       return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job: {:?}", category)));
     },
@@ -53,33 +59,41 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result<JobArgs, ProcessSingl
     }
   };
 
-  let sd_model = match &inference_args.maybe_sd_model {
+  let sd_model_token = match &inference_args.maybe_sd_model_token {
     Some(args) => args,
     None => {
       return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!")));
     }
   };
 
+  let prompt = match &inference_args.maybe_prompt {
+    Some(args) => args,
+    None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no prompt!")));
+    }
+  };
 
-  // let preprocess = match inference_args.maybe_preprocess {
-  //   None => Some("full".to_string()),
-  //   Some(Preprocess::F) => Some("full".to_string()),
-  //   Some(Preprocess::EF) => Some("extfull".to_string()),
-  //   Some(Preprocess::C) => Some("crop".to_string()),
-  //   Some(Preprocess::EC) => Some("extcrop".to_string()),
-  // };
-  //
-  // let enhancer = match inference_args.maybe_face_enhancer {
-  //   None => None,
-  //   Some(FaceEnhancer::G) => Some("gfpgan".to_string()),
-  //   Some(FaceEnhancer::R) => Some("RestoreFormer".to_string()),
-  // };
-  //
-  // let mut width = inference_args.maybe_resize_width;
-  // let mut height = inference_args.maybe_resize_height;
+  let a_prompt = match &inference_args.maybe_a_prompt {
+      Some(args) => args,
+      None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no a prompt!")));
+      }
+  };
+
+  let n_prompt = match &inference_args.maybe_n_prompt {
+      Some(args) => args,
+      None => {
+      return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no n prompt!")));
+      }
+  };
 
   Ok(JobArgs {
     video_source,
-    sd_model,
+    sd_model_token,
+    lora_model_token: &inference_args.maybe_lora_model_token,
+    prompt,
+    a_prompt,
+    n_prompt,
+    seed: inference_args.maybe_seed,
   })
 }

crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs

@@ -20,6 +20,7 @@ use mysql_queries::payloads::generic_inference_args::videofilter_payload::{Reren
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
 use tokens::tokens::generic_inference_jobs::InferenceJobToken;
 use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::model_weights::ModelWeightToken;
 use tokens::tokens::users::UserToken;
 
 use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
@@ -38,8 +39,13 @@ const ROUTING_TAG_HEADER_NAME : &str = "routing-tag";
 pub struct EnqueueRerenderAnimationRequest {
     uuid_idempotency_token: String,
 
+    prompt: Option<String>,  // prompt for the new video
+    a_prompt: Option<String>,  // prompt describing the old video
+    n_prompt: Option<String>,  // negative prompt
     video_source: Option<MediaFileToken>,
-    sd_model: String,
+    sd_model_token: Option<ModelWeightToken>,
+    lora_model_token: Option<ModelWeightToken>,
+    seed: Option<i32>,
     creator_set_visibility: Option<Visibility>,
 }
 
@@ -158,13 +164,15 @@ pub async fn enqueue_rerender_animation_handler(
 
     // TODO(bt): CHECK DATABASE FOR TOKENS!
 
-    let video_source: VideofilterVideoSource = {
-        if let Some(ref token) = request.video_source {
-            VideofilterVideoSource::media_file_token(token.as_str())
-        } else {
-            return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string()));
-        }
-    };
+    let video_source: VideofilterVideoSource = request.video_source
+        .as_ref()
+        .map(|token| VideofilterVideoSource::media_file_token(token.as_str()))
+        .ok_or_else(|| EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string()))?;
+
+    let sd_model_token: ModelWeightToken = request.sd_model_token
+        .as_ref()
+        .map(|token| ModelWeightToken::new_from_str(token.as_str()))
+        .ok_or_else(|| EnqueueRerenderAnimationError::BadInput("sd model not fully specified".to_string()))?;
 
     let ip_address = get_request_ip(&http_request);
 
@@ -178,14 +186,19 @@ pub async fn enqueue_rerender_animation_handler(
 
     let inference_args = RerenderArgs {
         maybe_video_source: Some(video_source),
-        maybe_sd_model: Some(request.sd_model.clone()),
+        maybe_sd_model_token: Some(sd_model_token),
+        maybe_lora_model_token: request.lora_model_token.clone(),
+        maybe_prompt: request.prompt.clone(),
+        maybe_a_prompt: request.a_prompt.clone(),
+        maybe_n_prompt: request.n_prompt.clone(),
+        maybe_seed: request.seed.clone(),
     };
 
     info!("Creating rerender animation job record...");
 
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
         uuid_idempotency_token: &request.uuid_idempotency_token,
-        inference_category: InferenceCategory::RerenderAVideo,
+        inference_category: InferenceCategory::VideoFilter,
         maybe_model_type: Some(InferenceModelType::RerenderAVideo), // NB: Model is static during inference
         maybe_model_token: None, // NB: Model is static during inference
         maybe_input_source_token: None, // TODO: Introduce a second foreign key ?
@@ -193,7 +206,7 @@ pub async fn enqueue_rerender_animation_handler(
         maybe_raw_inference_text: None, // No text
         maybe_max_duration_seconds: None,
         maybe_inference_args: Some(GenericInferenceArgs {
-            inference_category: Some(InferenceCategoryAbbreviated::RerenderAVideo),
+            inference_category: Some(InferenceCategoryAbbreviated::VideoFilter),
             args: Some(PolymorphicInferenceArgs::Rr(inference_args)),
         }),
         maybe_creator_user_token: maybe_user_token.as_ref(),

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs

@@ -256,7 +256,7 @@ fn record_to_payload(
               .get_full_object_path_str()
               .to_string()
         }
-        InferenceCategory::RerenderAVideo => {
+        InferenceCategory::VideoFilter => {
           MediaFileBucketPath::from_object_hash(
             &result_details.public_bucket_location_or_hash,
             result_details.maybe_media_file_public_bucket_prefix.as_deref(),