stubs for animation job type

.idea/dictionaries/bt.xml

@@ -21,6 +21,7 @@
       <w>hifigan</w>
       <w>hparams</w>
       <w>huggingface</w>
+      <w>lipsync</w>
       <w>pathbuf</w>
       <w>replayable</w>
       <w>retryably</w>

crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs

@@ -16,6 +16,10 @@ use strum::EnumIter;
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
 pub enum InferenceCategory {
+  /// Eg. SadTalker and possibly Wav2Lip
+  #[serde(rename = "lipsync_animation")]
+  LipsyncAnimation,
+
   #[serde(rename = "text_to_speech")]
   TextToSpeech,
 
@@ -31,6 +35,7 @@ impl_mysql_enum_coders!(InferenceCategory);
 impl InferenceCategory {
   pub fn to_str(&self) -> &'static str {
     match self {
+      Self::LipsyncAnimation => "lipsync_animation",
       Self::TextToSpeech => "text_to_speech",
       Self::VoiceConversion => "voice_conversion",
     }
@@ -38,6 +43,7 @@ impl InferenceCategory {
 
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
+      "lipsync_animation" => Ok(Self::LipsyncAnimation),
       "text_to_speech" => Ok(Self::TextToSpeech),
       "voice_conversion" => Ok(Self::VoiceConversion),
       _ => Err(format!("invalid value: {:?}", value)),
@@ -48,6 +54,7 @@ impl InferenceCategory {
     // NB: BTreeSet is sorted
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
+      Self::LipsyncAnimation,
       Self::TextToSpeech,
       Self::VoiceConversion,
     ])
@@ -61,18 +68,21 @@ mod tests {
 
   #[test]
   fn test_serialization() {
+    assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation");
     assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech");
     assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion");
   }
 
   #[test]
   fn to_str() {
+    assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation");
     assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech");
     assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion");
   }
 
   #[test]
   fn from_str() {
+    assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation);
     assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech);
     assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion);
   }
@@ -81,7 +91,8 @@ mod tests {
   fn all_variants() {
     // Static check
     let mut variants = InferenceCategory::all_variants();
-    assert_eq!(variants.len(), 2);
+    assert_eq!(variants.len(), 3);
+    assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation));
     assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech));
     assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion));
     assert_eq!(variants.pop_first(), None);

crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs

@@ -21,6 +21,10 @@ pub enum InferenceModelType {
   #[serde(rename = "rvc_v2")]
   RvcV2,
 
+  // NB: sad_talker does use user-supplied models, so there is no "model token"
+  #[serde(rename = "sad_talker")]
+  SadTalker,
+
   #[serde(rename = "so_vits_svc")]
   SoVitsSvc,
 
@@ -41,6 +45,7 @@ impl InferenceModelType {
   pub fn to_str(&self) -> &'static str {
     match self {
       Self::RvcV2 => "rvc_v2",
+      Self::SadTalker => "sad_talker",
       Self::SoVitsSvc => "so_vits_svc",
       Self::Tacotron2 => "tacotron2",
       Self::Vits => "vits",
@@ -50,6 +55,7 @@ impl InferenceModelType {
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
       "rvc_v2" => Ok(Self::RvcV2),
+      "sad_talker" => Ok(Self::SadTalker),
       "so_vits_svc" => Ok(Self::SoVitsSvc),
       "tacotron2" => Ok(Self::Tacotron2),
       "vits" => Ok(Self::Vits),
@@ -62,6 +68,7 @@ impl InferenceModelType {
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
       InferenceModelType::RvcV2,
+      InferenceModelType::SadTalker,
       InferenceModelType::SoVitsSvc,
       InferenceModelType::Tacotron2,
       InferenceModelType::Vits,
@@ -77,6 +84,7 @@ mod tests {
   #[test]
   fn test_serialization() {
     assert_serialization(InferenceModelType::RvcV2, "rvc_v2");
+    assert_serialization(InferenceModelType::SadTalker, "sad_talker");
     assert_serialization(InferenceModelType::SoVitsSvc, "so_vits_svc");
     assert_serialization(InferenceModelType::Tacotron2, "tacotron2");
     assert_serialization(InferenceModelType::Vits, "vits");
@@ -85,6 +93,7 @@ mod tests {
   #[test]
   fn to_str() {
     assert_eq!(InferenceModelType::RvcV2.to_str(), "rvc_v2");
+    assert_eq!(InferenceModelType::SadTalker.to_str(), "sad_talker");
     assert_eq!(InferenceModelType::SoVitsSvc.to_str(), "so_vits_svc");
     assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2");
     assert_eq!(InferenceModelType::Vits.to_str(), "vits");
@@ -93,6 +102,7 @@ mod tests {
   #[test]
   fn from_str() {
     assert_eq!(InferenceModelType::from_str("rvc_v2").unwrap(), InferenceModelType::RvcV2);
+    assert_eq!(InferenceModelType::from_str("sad_talker").unwrap(), InferenceModelType::SadTalker);
     assert_eq!(InferenceModelType::from_str("so_vits_svc").unwrap(), InferenceModelType::SoVitsSvc);
     assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2);
     assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits);
@@ -102,8 +112,9 @@ mod tests {
   fn all_variants() {
     // Static check
     let mut variants = InferenceModelType::all_variants();
-    assert_eq!(variants.len(), 4);
+    assert_eq!(variants.len(), 5);
     assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
+    assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
     assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
     assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
     assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));

crates/lib/mysql_queries/src/mediators/firehose_publisher.rs

@@ -32,6 +32,10 @@ enum FirehoseEvent {
   VcInferenceStarted,
   VcInferenceCompleted,
 
+  // SadTalker, not Wav2Lip
+  LipsyncAnimationStarted,
+  LipsyncAnimationCompleted,
+
   GenericDownloadStarted,
   GenericDownloadCompleted,
 
@@ -64,6 +68,8 @@ impl FirehoseEvent {
       FirehoseEvent::W2lInferenceCompleted => "w2l_inference_completed",
       FirehoseEvent::VcInferenceStarted => "vc_inference_started",
       FirehoseEvent::VcInferenceCompleted => "vc_inference_completed",
+      FirehoseEvent::LipsyncAnimationStarted => "lipsync_animation_started",
+      FirehoseEvent::LipsyncAnimationCompleted => "lipsync_animation_completed",
       FirehoseEvent::GenericDownloadStarted => "generic_download_started",
       FirehoseEvent::GenericDownloadCompleted => "generic_download_completed",
       FirehoseEvent::MediaUploaded => "media_uploaded",
@@ -220,6 +226,16 @@ impl FirehosePublisher {
     Ok(())
   }
 
+  pub async fn enqueue_lipsync_animation(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken) -> AnyhowResult<()> {
+    let _record_id = self.insert(
+      FirehoseEvent::LipsyncAnimationStarted,
+      maybe_user_token.map(|u| u.as_str()),
+      Some(inference_job_token.as_str()),
+      Some(inference_job_token.as_str()),
+    ).await?;
+    Ok(())
+  }
+
   pub async fn enqueue_generic_download(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> {
     let _record_id = self.insert(
       FirehoseEvent::GenericDownloadStarted,

crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs

@@ -1,8 +1,6 @@
+use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use errors::AnyhowResult;
-use tokens::files::media_upload::MediaUploadToken;
-use tokens::tokens::tts_models::TtsModelToken;
-use tokens::voice_conversion::model::VoiceConversionModelToken;
 
 /// Used to encode extra state for the `generic_inference_jobs` table.
 /// This should act somewhat like a serialized protobuf stored inside a record.
@@ -23,6 +21,10 @@ pub struct GenericInferenceArgs {
 /// Do not change the values.
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
 pub enum InferenceCategoryAbbreviated {
+  #[serde(rename = "la")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(alias = "lipsync_animation")]
+  LipsyncAnimation,
+
   #[serde(rename = "tts")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
   #[serde(alias = "text_to_speech")]
   TextToSpeech,
@@ -44,6 +46,10 @@ pub enum FundamentalFrequencyMethodForJob {
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum PolymorphicInferenceArgs {
+  /// Lipsync Animation (Short name to save  space when serializing)
+  /// This is SadTalker, not Wav2Lip.
+  La (LipsyncArgs),
+
   /// Text to speech. (Short name to save space when serializing.)
   Tts {
     // No arguments yet.
@@ -74,6 +80,7 @@ pub enum PolymorphicInferenceArgs {
   },
 }
 
+
 impl GenericInferenceArgs {
 
   pub fn from_json(json: &str) -> AnyhowResult<Self> {
@@ -88,6 +95,7 @@ impl GenericInferenceArgs {
 impl InferenceCategoryAbbreviated {
   pub fn from_inference_category(category: InferenceCategory) -> Self {
     match category {
+      InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation,
       InferenceCategory::TextToSpeech => Self::TextToSpeech,
       InferenceCategory::VoiceConversion => Self::VoiceConversion,
     }
@@ -95,6 +103,7 @@ impl InferenceCategoryAbbreviated {
 
   pub fn to_inference_category(self) -> InferenceCategory {
     match self {
+      Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation,
       Self::TextToSpeech => InferenceCategory::TextToSpeech,
       Self::VoiceConversion => InferenceCategory::VoiceConversion,
     }
@@ -103,11 +112,27 @@ impl InferenceCategoryAbbreviated {
 
 #[cfg(test)]
 mod tests {
-  use crate::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
-  use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
-  use tokens::files::media_upload::MediaUploadToken;
-  use tokens::tokens::tts_models::TtsModelToken;
-  use tokens::voice_conversion::model::VoiceConversionModelToken;
+  use crate::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+  use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
+
+  #[test]
+  fn typical_lipsync_animation_args_serialize() {
+    let args = GenericInferenceArgs {
+      inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation),
+      args: Some(PolymorphicInferenceArgs::La(LipsyncArgs {
+        maybe_audio_source: Some(LipsyncAnimationAudioSource::U("foo".to_string())),
+        maybe_image_source: Some(LipsyncAnimationImageSource::F("bar".to_string())),
+      })),
+    };
+
+    let json = serde_json::ser::to_string(&args).unwrap();
+
+    // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating.
+    assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":{"U":"foo"},"i":{"F":"bar"}}}}"#.to_string());
+
+    // NB: Make sure we don't overflow the DB field capacity (TEXT column).
+    assert!(json.len() < 1000);
+  }
 
   #[test]
   fn typical_tts_args_serialize() {

crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs

@@ -0,0 +1,125 @@
+//! Arguments for lipsync inferences
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct LipsyncArgs {
+  #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub maybe_audio_source: Option<LipsyncAnimationAudioSource>,
+
+  #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub maybe_image_source: Option<LipsyncAnimationImageSource>,
+}
+
+/// Audio sources can be one of several:
+///  - F: media_files (todo)
+///  - U: media_uploads (legacy)
+///  - T: tts_results (legacy)
+///  - V: voice_conversion_results (legacy)
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum LipsyncAnimationAudioSource {
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media File Token (media_files table)
+  /// Serde cannot yet rename enum variants.
+  F(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media Upload Token (media_uploads table)
+  /// Serde cannot yet rename enum variants.
+  U(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// TTS Result Token (tts_results table)
+  /// Serde cannot yet rename enum variants.
+  T(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Voice Conversion Result Token (voice_conversion_results table)
+  /// Serde cannot yet rename enum variants.
+  V(String),
+}
+
+// TODO: Hydrate to a public enum with easier to read names.
+impl LipsyncAnimationAudioSource {
+  pub fn media_file_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::F(token.to_string())
+  }
+  pub fn media_upload_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::U(token.to_string())
+  }
+  pub fn tts_result_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::T(token.to_string())
+  }
+  pub fn voice_conversion_result_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::V(token.to_string())
+  }
+}
+
+/// Video sources can be one of several:
+///  - F: media_files (todo)
+///  - U: media_uploads (legacy)
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum LipsyncAnimationImageSource {
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media File Token (media_files table)
+  /// Serde cannot yet rename enum variants.
+  F(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media Upload Token (media_uploads table)
+  /// Serde cannot yet rename enum variants.
+  U(String),
+}
+
+impl LipsyncAnimationImageSource {
+  pub fn media_file_token(token: &str) -> Self {
+    LipsyncAnimationImageSource::F(token.to_string())
+  }
+  pub fn media_upload_token(token: &str) -> Self {
+    LipsyncAnimationImageSource::U(token.to_string())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
+
+  #[test]
+  fn test_media_file() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::media_file_token("audio_media_file")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_file_token("image_media_file")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"F":"audio_media_file"},"i":{"F":"image_media_file"}}"#.to_string());
+  }
+
+  #[test]
+  fn test_media_upload() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::media_upload_token("audio_media_upload")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"U":"audio_media_upload"},"i":{"U":"image_media_upload"}}"#.to_string());
+  }
+
+  #[test]
+  fn test_tts_result() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::tts_result_token("audio_tts_result")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"T":"audio_tts_result"},"i":{"U":"image_media_upload"}}"#.to_string());
+  }
+  #[test]
+  fn test_voice_conversion_result() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::voice_conversion_result_token("audio_voice_conversion_result")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"V":"audio_voice_conversion_result"},"i":{"U":"image_media_upload"}}"#.to_string());
+  }
+}

crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs

@@ -0,0 +1,2 @@
+pub mod generic_inference_args;
+pub mod lipsync_payload;

crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs

@@ -13,7 +13,7 @@ use std::future::Future;
 use std::path::Path;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use tokens::jobs::inference::InferenceJobToken;
-use crate::payloads::generic_inference_args::GenericInferenceArgs;
+use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs;
 
 /// table: generic_inference_jobs
 #[derive(Debug)]
@@ -26,7 +26,7 @@ pub struct AvailableInferenceJob {
   // Inference class
   pub inference_category: InferenceCategory,
 
-  pub maybe_model_type: Option<String>, // TODO: Strongly type.
+  pub maybe_model_type: Option<InferenceModelType>,
   pub maybe_model_token: Option<String>,
 
   pub maybe_input_source_token: Option<String>,
@@ -115,7 +115,11 @@ pub async fn list_available_generic_inference_jobs(
               .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
           inference_category: InferenceCategory::from_str(&record.inference_category)
               .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
-          maybe_model_type: record.maybe_model_type,
+          maybe_model_type: record.maybe_model_type
+              .as_deref()
+              .map(|model_type| InferenceModelType::from_str(model_type))
+              .transpose()
+              .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
           maybe_model_token: record.maybe_model_token,
           maybe_input_source_token: record.maybe_input_source_token,
           maybe_input_source_token_type: record.maybe_input_source_token_type,

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -120,6 +120,7 @@ WHERE jobs.token = ?
   };
 
   let maybe_model_title = match record.inference_category {
+    InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
     InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
     InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
   };
@@ -127,6 +128,7 @@ WHERE jobs.token = ?
   // NB: A bit of a hack. We store TTS results with a full path.
   // Going forward, all other record types will store a hash.
   let (bucket_path_is_hash, maybe_public_bucket_hash) = match record.inference_category {
+    InferenceCategory::LipsyncAnimation => (true, Some("todo")), // TODO - these values are wrong
     InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
     InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
   };

crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs

@@ -1,5 +1,5 @@
 use anyhow::anyhow;
-use crate::payloads::generic_inference_args::GenericInferenceArgs;
+use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;

crates/service/job/inference_job/src/job/job_loop/process_single_job.rs

@@ -14,6 +14,7 @@ use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_s
 use r2d2_redis::redis::Commands;
 use redis_common::redis_keys::RedisKeys;
 use std::time::Instant;
+use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
 
 pub async fn process_single_job(
   job_dependencies: &JobDependencies,
@@ -167,6 +168,9 @@ async fn do_process_single_job(
   let mut maybe_result_entity : Option<ResultEntity>;
 
   let job_success_result = match job.inference_category {
+    InferenceCategory::LipsyncAnimation => {
+      process_single_lipsync_job(job_dependencies, job).await?
+    }
     InferenceCategory::TextToSpeech => {
       process_single_tts_job(job_dependencies, job).await?
     }

crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs

@@ -0,0 +1,2 @@
+pub mod process_single_lipsync_job;
+pub mod sad_talker;

crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs

@@ -0,0 +1,36 @@
+use anyhow::anyhow;
+use crate::job::job_loop::job_success_result::JobSuccessResult;
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs;
+use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs;
+use crate::job::job_types::vc::{rvc_v2, so_vits_svc};
+use crate::job_dependencies::JobDependencies;
+use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference;
+use std::time::Duration;
+use log::{error, info};
+use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
+use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
+use tokens::files::media_upload::MediaUploadToken;
+
+pub async fn process_single_lipsync_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let job_success_result = match job.maybe_model_type {
+    Some(InferenceModelType::SadTalker) => {
+      //rvc_v2::process_job::process_job(RvcV2ProcessJobArgs {
+      //  job_dependencies,
+      //  job,
+      //  vc_model: &vc_model,
+      //  media_upload_token: &media_upload_token,
+      //  media_upload: &media_upload,
+      //}).await?
+      return Err(ProcessSingleJobError::Other(anyhow!("not yet implemented")))
+    }
+    Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))),
+    None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))),
+  };
+
+  Ok(job_success_result)
+}
+

crates/service/job/inference_job/src/job/job_types/mod.rs

@@ -1,2 +1,3 @@
+pub mod lipsync;
 pub mod tts;
 pub mod vc;

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs

@@ -15,9 +15,9 @@ use filesys::file_size::file_size;
 use log::{error, info, warn};
 use media::decode_basic_audio_info::decode_basic_audio_file_info;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
-use mysql_queries::payloads::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::media_uploads::get_media_upload_for_inference::{get_media_upload_for_inference, MediaUploadRecordForInference};
+use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
 use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference;
 use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs};
 use std::time::Instant;
@@ -149,6 +149,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
   let auto_predict_f0 = maybe_args
       .map(|args| match args {
         PolymorphicInferenceArgs::Tts { .. } => None,
+        PolymorphicInferenceArgs::La(_) => None,
         PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0,
       })
       .flatten()
@@ -157,6 +158,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
   let maybe_transpose = maybe_args
       .map(|args| match args {
         PolymorphicInferenceArgs::Tts { .. } => None,
+        PolymorphicInferenceArgs::La(_) => None,
         PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose,
       })
       .flatten();
@@ -164,6 +166,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
   let maybe_override_f0_method = maybe_args
       .map(|args| match args {
         PolymorphicInferenceArgs::Tts { .. } => None,
+        PolymorphicInferenceArgs::La(_) => None,
         PolymorphicInferenceArgs::Vc { override_f0_method, .. } =>
           *override_f0_method,
       })

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs

@@ -1,18 +1,17 @@
 use anyhow::anyhow;
 use container_common::anyhow_result::AnyhowResult;
+use crate::job::job_loop::command_exit_status::CommandExitStatus;
 use filesys::path_to_string::path_to_string;
 use log::info;
-use mysql_queries::payloads::generic_inference_args::FundamentalFrequencyMethodForJob;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::FundamentalFrequencyMethodForJob;
 use once_cell::sync::Lazy;
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
-use std::fs::OpenOptions;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
-use subprocess::{ExitStatus, Popen, PopenConfig, Redirection};
+use subprocess::{Popen, PopenConfig};
 use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions};
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
 
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!

crates/service/web/storyteller_web/src/configs/plans/plan.rs

@@ -42,6 +42,10 @@ pub struct Plan {
     /// Whether this plan does not exist in Stripe and only exists as defined by this server.
     is_synthetic_plan: bool,
 
+    // ========== Features for lipsync (SadTalker, not Wav2Lip) ==========
+
+    lipsync_requires_frontend_keepalive: bool,
+
     // ========== Features for TTS ==========
 
     tts_base_priority_level: u8,
@@ -83,6 +87,7 @@ impl Plan {
             stripe_price_id: builder.stripe_price_id.clone(),
             is_development_plan: builder.is_development_plan,
             is_synthetic_plan: builder.is_synthetic_plan,
+            lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive,
             tts_base_priority_level: builder.tts_base_priority_level,
             tts_max_duration: builder.tts_max_duration,
             tts_max_character_length: builder.tts_max_character_length,
@@ -133,6 +138,10 @@ impl Plan {
         self.is_synthetic_plan
     }
 
+    pub fn lipsync_requires_frontend_keepalive(&self) -> bool {
+        self.lipsync_requires_frontend_keepalive
+    }
+
     pub fn tts_base_priority_level(&self) -> u8 {
         self.tts_base_priority_level
     }
@@ -169,6 +178,10 @@ pub struct PlanBuilder {
     is_development_plan: bool,
     is_synthetic_plan: bool,
 
+    // ========== Features for lipsync (SadTalker, not Wav2Lip) ==========
+
+    lipsync_requires_frontend_keepalive: bool,
+
     // ========== Features for TTS ==========
 
     tts_base_priority_level: u8,
@@ -212,6 +225,9 @@ impl PlanBuilder {
             is_development_plan: false,
             is_synthetic_plan: false,
 
+            // Lipsync (SadTalker, not Wav2Lip)
+            lipsync_requires_frontend_keepalive: true,
+
             // TTS
             tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL,
             tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS),
@@ -272,6 +288,11 @@ impl PlanBuilder {
         self
     }
 
+    pub fn lipsync_requires_frontend_keepalive(mut self, value: bool) -> Self {
+        self.lipsync_requires_frontend_keepalive = value;
+        self
+    }
+
     pub fn tts_base_priority_level(mut self, value: u8) -> Self {
         self.tts_base_priority_level = value;
         self

crates/service/web/storyteller_web/src/configs/plans/plan_list.rs

@@ -15,6 +15,7 @@ pub static FREE_LOGGED_OUT_PLAN : Lazy<Plan> = Lazy::new(|| {
         .tts_max_character_length(1024)
         .web_vc_base_priority_level(FAKEYOU_ANONYMOUS_PRIORITY_LEVEL)
         .web_vc_requires_frontend_keepalive(true)
+        .lipsync_requires_frontend_keepalive(true)
         .build()
 });
 
@@ -28,6 +29,7 @@ pub static FREE_LOGGED_OUT_FIRST_TRY_PLAN : Lazy<Plan> = Lazy::new(|| {
         .tts_max_character_length(1024)
         .web_vc_base_priority_level(FAKEYOU_LOGGED_IN_PRIORITY_LEVEL) // NB: Same as logged-in free users.
         .web_vc_requires_frontend_keepalive(true)
+        .lipsync_requires_frontend_keepalive(true)
         .build()
 });
 
@@ -41,6 +43,7 @@ pub static FREE_LOGGED_IN_PLAN : Lazy<Plan> = Lazy::new(|| {
         .tts_max_character_length(1024)
         .web_vc_base_priority_level(FAKEYOU_LOGGED_IN_PRIORITY_LEVEL)
         .web_vc_requires_frontend_keepalive(true)
+        .lipsync_requires_frontend_keepalive(true)
         .build()
 });
 
@@ -56,6 +59,7 @@ pub static LOYALTY_PLANS : Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(30)
         .web_vc_base_priority_level(2)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans
@@ -77,6 +81,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_character_length(2048)
         .tts_max_duration_seconds(30)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("development_fakeyou_pro")
@@ -89,6 +94,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_character_length(2048)
         .tts_max_duration_seconds(60 * 2)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("development_fakeyou_elite")
@@ -101,6 +107,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_character_length(2048)
         .tts_max_duration_seconds(60 * 5)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans
@@ -122,6 +129,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(30)
         .web_vc_base_priority_level(20)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("fakeyou_pro")
@@ -134,6 +142,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(60 * 2)
         .web_vc_base_priority_level(30)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("fakeyou_elite")
@@ -146,6 +155,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(60 * 5)
         .web_vc_base_priority_level(40)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     // ========== Spanish plans ==========
@@ -158,6 +168,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_base_priority_level(10)
         .web_vc_base_priority_level(10)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .tts_max_duration_seconds(30)
         .build());
 
@@ -405,6 +416,7 @@ mod test {
     fn test_assert_web_vc_keepalive_forced() {
         FREE_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| {
             assert!(plan.web_vc_requires_frontend_keepalive());
+            assert!(plan.lipsync_requires_frontend_keepalive());
         });
     }
 
@@ -412,9 +424,11 @@ mod test {
     fn test_assert_web_vc_no_keepalive() {
         LOYALTY_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| {
             assert!(!plan.web_vc_requires_frontend_keepalive());
+            assert!(!plan.lipsync_requires_frontend_keepalive());
         });
         PRODUCTION_PREMIUM_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| {
             assert!(!plan.web_vc_requires_frontend_keepalive());
+            assert!(!plan.lipsync_requires_frontend_keepalive());
         });
     }
 

crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs

@@ -0,0 +1,323 @@
+// NB: Incrementally getting rid of build warnings...
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+use actix_web::error::ResponseError;
+use actix_web::http::StatusCode;
+use actix_web::{web, HttpResponse, HttpRequest};
+use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
+use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie;
+use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
+use crate::server_state::ServerState;
+use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::common::visibility::Visibility;
+use http_server_common::request::get_request_header_optional::get_request_header_optional;
+use http_server_common::request::get_request_ip::get_request_ip;
+use log::{info, warn};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
+use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job};
+use std::sync::Arc;
+use tokens::files::media_upload::MediaUploadToken;
+use tokens::jobs::inference::InferenceJobToken;
+use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::voice_conversion_results::VoiceConversionResultToken;
+use tokens::users::user::UserToken;
+use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL;
+
+/// Debug requests can get routed to special "debug-only" workers, which can
+/// be used to trial new code, run debugging, etc.
+const DEBUG_HEADER_NAME : &'static str = "enable-debug-mode";
+
+/// The routing tag header can send workloads to particular k8s hosts.
+/// This is useful for catching the live logs or intercepting the job.
+const ROUTING_TAG_HEADER_NAME : &'static str = "routing-tag";
+
+#[derive(Deserialize)]
+pub struct EnqueueLipsyncAnimationRequest {
+  uuid_idempotency_token: String,
+
+  audio_source: AudioSource,
+  image_source: ImageSource,
+
+  creator_set_visibility: Option<Visibility>,
+  is_storyteller_demo: Option<bool>,
+
+  /// SadTalker: parameter to animate the full body.
+  animate_full_body: Option<bool>,
+}
+
+/// Treated as an enum. Only one of these may be set.
+#[derive(Deserialize)]
+pub struct AudioSource {
+  maybe_media_file_token: Option<MediaFileToken>,
+  maybe_media_upload_token: Option<MediaUploadToken>,
+  maybe_tts_result_token: Option<String>,
+  maybe_voice_conversion_result_token: Option<VoiceConversionResultToken>,
+}
+
+/// Treated as an enum. Only one of these may be set.
+#[derive(Deserialize)]
+pub struct ImageSource {
+  maybe_media_file_token: Option<MediaFileToken>,
+  maybe_media_upload_token: Option<MediaUploadToken>,
+}
+
+#[derive(Serialize)]
+pub struct EnqueueLipsyncAnimationSuccessResponse {
+  pub success: bool,
+  pub inference_job_token: InferenceJobToken,
+}
+
+#[derive(Debug)]
+pub enum EnqueueLipsyncAnimationError {
+  BadInput(String),
+  NotAuthorized,
+  ServerError,
+  RateLimited,
+}
+
+impl ResponseError for EnqueueLipsyncAnimationError {
+  fn status_code(&self) -> StatusCode {
+    match *self {
+      EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST,
+      EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED,
+      EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
+      EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
+    }
+  }
+
+  fn error_response(&self) -> HttpResponse {
+    let error_reason = match self {
+      EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(),
+      EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(),
+      EnqueueLipsyncAnimationError::ServerError => "server error".to_string(),
+      EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(),
+    };
+
+    to_simple_json_error(&error_reason, self.status_code())
+  }
+}
+
+// NB: Not using derive_more::Display since Clion doesn't understand it.
+impl std::fmt::Display for EnqueueLipsyncAnimationError {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "{:?}", self)
+  }
+}
+
+pub async fn enqueue_lipsync_animation_handler(
+  http_request: HttpRequest,
+  request: web::Json<EnqueueLipsyncAnimationRequest>,
+  server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, EnqueueLipsyncAnimationError>
+{
+  let mut maybe_user_token : Option<UserToken> = None;
+  let mut priority_level ;
+  let disable_rate_limiter = false; // NB: Careful!
+
+  let mut mysql_connection = server_state.mysql_pool
+      .acquire()
+      .await
+      .map_err(|err| {
+        warn!("MySql pool error: {:?}", err);
+        EnqueueLipsyncAnimationError::ServerError
+      })?;
+
+  // ==================== USER SESSION ==================== //
+
+  let maybe_user_session = server_state
+      .session_checker
+      .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection)
+      .await
+      .map_err(|e| {
+        warn!("Session checker error: {:?}", e);
+        EnqueueLipsyncAnimationError::ServerError
+      })?;
+
+  if let Some(user_session) = maybe_user_session.as_ref() {
+    maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token));
+  }
+
+  // TODO: Plan should handle "first anonymous use" and "investor" cases.
+  let plan = get_correct_plan_for_session(
+    server_state.server_environment,
+    maybe_user_session.as_ref());
+
+  // TODO: Separate priority for animation.
+  priority_level = plan.web_vc_base_priority_level();
+
+  // ==================== INVESTOR PRIORITY ==================== //
+
+  // TODO/TEMP: Give investors even more priority
+  let mut is_investor = false;
+
+  {
+    // TODO/TEMP: The storyteller.io website's AJAX calls will set this.
+    //  This is just for the YCombinator demo.
+    match request.is_storyteller_demo {
+      Some(true) => {
+        is_investor = true;
+      },
+      _ => {},
+    };
+
+    // TODO/TEMP: The storyteller.io website will redirect and establish this cookie.
+    //  This is just for the YCombinator demo.
+    if request_has_demo_cookie(&http_request) {
+      is_investor = true;
+    }
+
+    if is_investor {
+      priority_level = FAKEYOU_INVESTOR_PRIORITY_LEVEL;
+    }
+  }
+
+  // ==================== DEBUG MODE + ROUTING TAG ==================== //
+
+  let is_debug_request =
+      get_request_header_optional(&http_request, DEBUG_HEADER_NAME)
+          .is_some();
+
+  let maybe_routing_tag=
+      get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME)
+          .map(|routing_tag| routing_tag.trim().to_string());
+
+  // ==================== RATE LIMIT ==================== //
+
+  if !disable_rate_limiter {
+    let mut rate_limiter = match maybe_user_session {
+      None => &server_state.redis_rate_limiters.logged_out,
+      Some(ref user) => {
+        if user.role.is_banned {
+          return Err(EnqueueLipsyncAnimationError::NotAuthorized);
+        }
+        &server_state.redis_rate_limiters.logged_in
+      },
+    };
+
+    // TODO/TEMP
+    if is_investor {
+      rate_limiter = &server_state.redis_rate_limiters.logged_in;
+    }
+
+    if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
+      return Err(EnqueueLipsyncAnimationError::RateLimited);
+    }
+  }
+
+  // ==================== LOOK UP MODEL INFO ==================== //
+
+  // TODO(bt): CHECK DATABASE FOR TOKENS!
+
+  let audio_source = {
+    if let Some(ref token) = request.audio_source.maybe_media_file_token {
+      LipsyncAnimationAudioSource::media_file_token(token.as_str())
+    } else if let Some(ref token) = request.audio_source.maybe_media_upload_token {
+      LipsyncAnimationAudioSource::media_upload_token(token.as_str())
+    } else if let Some(ref token) = request.audio_source.maybe_tts_result_token {
+      LipsyncAnimationAudioSource::tts_result_token(token)
+    } else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token {
+      LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str())
+    } else {
+      return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string()));
+    }
+  };
+
+  let image_source = {
+    if let Some(ref token) = request.image_source.maybe_media_file_token {
+      LipsyncAnimationImageSource::media_file_token(token.as_str())
+    } else if let Some(ref token) = request.image_source.maybe_media_upload_token {
+      LipsyncAnimationImageSource::media_upload_token(token.as_str())
+    } else {
+      return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string()));
+    }
+  };
+
+  // ==================== CHECK AND ENQUEUE ANIMATION ==================== //
+
+  //let mut redis = server_state.redis_pool
+  //    .get()
+  //    .map_err(|e| {
+  //      error!("redis error: {:?}", e);
+  //      EnqueueLipsyncAnimationError::ServerError
+  //    })?;
+
+  //let redis_count_key = RedisKeys::web_vc_model_usage_count(model_token.as_str());
+
+  //redis.incr(&redis_count_key, 1)
+  //    .map_err(|e| {
+  //      warn!("redis error: {:?}", e);
+  //      EnqueueLipsyncAnimationError::ServerError
+  //    })?;
+
+  let ip_address = get_request_ip(&http_request);
+
+  let maybe_user_preferred_visibility : Option<Visibility> = maybe_user_session
+      .as_ref()
+      .map(|user_session| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc
+
+  let set_visibility = request.creator_set_visibility
+      .or(maybe_user_preferred_visibility)
+      .unwrap_or(Visibility::Public);
+
+
+  info!("Creating lipsync animation job record...");
+
+  // TODO: other SadTalker arguments.
+  let maybe_args = Some(PolymorphicInferenceArgs::La(LipsyncArgs {
+    maybe_audio_source: Some(audio_source),
+    maybe_image_source: Some(image_source),
+  }));
+
+  let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
+    uuid_idempotency_token: &request.uuid_idempotency_token,
+    inference_category: InferenceCategory::LipsyncAnimation,
+    maybe_model_type: None, // NB: Model is static during inference
+    maybe_model_token: None, // NB: Model is static during inference
+    maybe_input_source_token: None, // TODO: Introduce a second foreign key ?
+    maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ?
+    maybe_raw_inference_text: None, // No text
+    maybe_inference_args: Some(GenericInferenceArgs {
+      inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation),
+      args: maybe_args,
+    }),
+    maybe_creator_user_token: maybe_user_token.as_ref(),
+    creator_ip_address: &ip_address,
+    creator_set_visibility: set_visibility,
+    priority_level,
+    requires_keepalive: plan.lipsync_requires_frontend_keepalive(),
+    is_debug_request,
+    maybe_routing_tag: maybe_routing_tag.as_deref(),
+    mysql_pool: &server_state.mysql_pool,
+  }).await;
+
+  let job_token = match query_result {
+    Ok((job_token, _id)) => job_token,
+    Err(err) => {
+      warn!("New generic inference job creation DB error: {:?}", err);
+      return Err(EnqueueLipsyncAnimationError::ServerError);
+    }
+  };
+
+  server_state.firehose_publisher.enqueue_lipsync_animation(
+    maybe_user_token.as_ref(),
+    &job_token)
+      .await
+      .map_err(|e| {
+        warn!("error publishing event: {:?}", e);
+        EnqueueLipsyncAnimationError::ServerError
+      })?;
+
+  let response = EnqueueLipsyncAnimationSuccessResponse {
+    success: true,
+    inference_job_token: job_token,
+  };
+
+  let body = serde_json::to_string(&response)
+      .map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?;
+
+  Ok(HttpResponse::Ok()
+      .content_type("application/json")
+      .body(body))
+}

crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs

@@ -0,0 +1 @@
+pub mod enqueue_lipsync_animation;

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs

@@ -216,6 +216,10 @@ pub async fn get_inference_job_status_handler(
     },
     maybe_result: record.maybe_result_details.map(|result_details| {
       let public_bucket_media_path = match inference_category {
+        InferenceCategory::LipsyncAnimation => {
+          // TODO
+          "TODO".to_string()
+        }
         InferenceCategory::TextToSpeech => {
           // NB: TTS results receive the legacy treatment where their table only reports the full bucket path
           result_details.public_bucket_location_or_hash

crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs

@@ -186,6 +186,10 @@ pub async fn handle_upload(
       "audio/opus" /* .opus */ => Some(MediaUploadType::Audio),
       "audio/x-flac" /* .flac */ => Some(MediaUploadType::Audio),
       "audio/x-wav" /* .wav */ => Some(MediaUploadType::Audio),
+      // Image
+      "image/jpeg" /* .jpg */ => Some(MediaUploadType::Image),
+      "image/png" /* .png */ => Some(MediaUploadType::Image),
+      "image/webp" /* .webp */ => Some(MediaUploadType::Image),
       // Video
       "video/mp4" /* .mp4 */ => Some(MediaUploadType::Video),
       "video/quicktime" /* .mov */ => Some(MediaUploadType::Video),
@@ -193,7 +197,7 @@ pub async fn handle_upload(
       _ => None,
     };
 
-    let do_decode = match mimetype {
+    let do_audio_decode = match mimetype {
       "audio/opus" => {
         // TODO/FIXME(bt, 2023-05-19): Symphonia is currently broken for Firefox's opus.
         //  We're on an off-master branch that may resolve the problem in the future, but for now
@@ -219,10 +223,14 @@ pub async fn handle_upload(
         //    right: `Ebml`: EBML element type must be checked before calling this function', /Users/bt/.cargo/git/checkouts/symphonia-8fbe6c90fc095688/e1a7009/symphonia-format-mkv/src/ebml.rs:335:9
         false
       }
+      // Also, don't decode images
+      "image/jpeg" => false,
+      "image/png" => false,
+      "image/webp" => false,
       _ => true,
     };
 
-    if do_decode && media_upload_type.is_some() {
+    if do_audio_decode && media_upload_type.is_some() {
       let basic_info = decode_basic_audio_bytes_info(
         bytes.as_ref(),
         Some(mimetype),

crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/mod.rs

@@ -1,4 +1,5 @@
 pub mod common;
 pub mod list_user_media_uploads_of_type;
 pub mod upload_audio;
+pub mod upload_image;
 pub mod upload_media;

crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_image.rs

@@ -0,0 +1,50 @@
+use actix_multipart::Multipart;
+use actix_web::{HttpRequest, HttpResponse, web};
+use crate::http_server::endpoints::media_uploads::common::handle_upload::handle_upload;
+use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError;
+use crate::server_state::ServerState;
+use once_cell::sync::Lazy;
+use std::collections::HashSet;
+use std::sync::Arc;
+use tokens::files::media_upload::MediaUploadToken;
+
+#[derive(Serialize)]
+pub struct UploadAudioSuccessResponse {
+  pub success: bool,
+  pub upload_token: MediaUploadToken,
+}
+
+static ALLOWED_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
+  HashSet::from([
+    "image/jpeg",
+    "image/png",
+    "image/webp",
+  ])
+});
+
+pub async fn upload_image_handler(
+  http_request: HttpRequest,
+  server_state: web::Data<Arc<ServerState>>,
+  mut multipart_payload: Multipart,
+) -> Result<HttpResponse, UploadError> {
+
+  let response = handle_upload(
+    &http_request,
+    &server_state,
+    multipart_payload,
+    &ALLOWED_MIME_TYPES).await?;
+
+  let media_upload_token = response.to_media_token();
+
+  let response = UploadAudioSuccessResponse {
+    success: true,
+    upload_token: media_upload_token,
+  };
+
+  let body = serde_json::to_string(&response)
+      .map_err(|e| UploadError::ServerError)?;
+
+  return Ok(HttpResponse::Ok()
+      .content_type("application/json")
+      .body(body));
+}

crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs

@@ -1,3 +1,4 @@
+pub mod animation;
 pub mod api_tokens;
 pub mod categories;
 pub mod comments;

crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs

@@ -20,7 +20,7 @@ use http_server_common::request::get_request_api_token::get_request_api_token;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
 use log::{info, warn};
-use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated};
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
 use mysql_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder;
 use mysql_queries::queries::tts::tts_models::get_tts_model::TtsModelRecord;

crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs

@@ -19,7 +19,7 @@ use enums::common::visibility::Visibility;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
 use log::{error, info, warn};
-use mysql_queries::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job};
 use mysql_queries::queries::voice_conversion::model_info_lite::get_voice_conversion_model_info_lite::get_voice_conversion_model_info_lite_with_connection;
 use r2d2_redis::redis::Commands;