enqueue animation endpoint + more plumbing

.idea/dictionaries/bt.xml

@@ -20,6 +20,7 @@
       <w>hifigan</w>
       <w>hparams</w>
       <w>huggingface</w>
+      <w>lipsync</w>
       <w>pathbuf</w>
       <w>replayable</w>
       <w>retryably</w>

crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs

@@ -16,6 +16,10 @@ use strum::EnumIter;
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
 pub enum InferenceCategory {
+  /// Eg. SadTalker and possibly Wav2Lip
+  #[serde(rename = "lipsync_animation")]
+  LipsyncAnimation,
+
   #[serde(rename = "text_to_speech")]
   TextToSpeech,
 
@@ -31,6 +35,7 @@ impl_mysql_enum_coders!(InferenceCategory);
 impl InferenceCategory {
   pub fn to_str(&self) -> &'static str {
     match self {
+      Self::LipsyncAnimation => "lipsync_animation",
       Self::TextToSpeech => "text_to_speech",
       Self::VoiceConversion => "voice_conversion",
     }
@@ -38,6 +43,7 @@ impl InferenceCategory {
 
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
+      "lipsync_animation" => Ok(Self::LipsyncAnimation),
       "text_to_speech" => Ok(Self::TextToSpeech),
       "voice_conversion" => Ok(Self::VoiceConversion),
       _ => Err(format!("invalid value: {:?}", value)),
@@ -48,6 +54,7 @@ impl InferenceCategory {
     // NB: BTreeSet is sorted
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
+      Self::LipsyncAnimation,
       Self::TextToSpeech,
       Self::VoiceConversion,
     ])
@@ -61,18 +68,21 @@ mod tests {
 
   #[test]
   fn test_serialization() {
+    assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation");
     assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech");
     assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion");
   }
 
   #[test]
   fn to_str() {
+    assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation");
     assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech");
     assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion");
   }
 
   #[test]
   fn from_str() {
+    assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation);
     assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech);
     assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion);
   }
@@ -81,7 +91,8 @@ mod tests {
   fn all_variants() {
     // Static check
     let mut variants = InferenceCategory::all_variants();
-    assert_eq!(variants.len(), 2);
+    assert_eq!(variants.len(), 3);
+    assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation));
     assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech));
     assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion));
     assert_eq!(variants.pop_first(), None);

crates/lib/mysql_queries/src/mediators/firehose_publisher.rs

@@ -32,6 +32,10 @@ enum FirehoseEvent {
   VcInferenceStarted,
   VcInferenceCompleted,
 
+  // SadTalker, not Wav2Lip
+  LipsyncAnimationStarted,
+  LipsyncAnimationCompleted,
+
   GenericDownloadStarted,
   GenericDownloadCompleted,
 
@@ -64,6 +68,8 @@ impl FirehoseEvent {
       FirehoseEvent::W2lInferenceCompleted => "w2l_inference_completed",
       FirehoseEvent::VcInferenceStarted => "vc_inference_started",
       FirehoseEvent::VcInferenceCompleted => "vc_inference_completed",
+      FirehoseEvent::LipsyncAnimationStarted => "lipsync_animation_started",
+      FirehoseEvent::LipsyncAnimationCompleted => "lipsync_animation_completed",
       FirehoseEvent::GenericDownloadStarted => "generic_download_started",
       FirehoseEvent::GenericDownloadCompleted => "generic_download_completed",
       FirehoseEvent::MediaUploaded => "media_uploaded",
@@ -220,6 +226,16 @@ impl FirehosePublisher {
     Ok(())
   }
 
+  pub async fn enqueue_lipsync_animation(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken) -> AnyhowResult<()> {
+    let _record_id = self.insert(
+      FirehoseEvent::LipsyncAnimationStarted,
+      maybe_user_token.map(|u| u.as_str()),
+      Some(inference_job_token.as_str()),
+      Some(inference_job_token.as_str()),
+    ).await?;
+    Ok(())
+  }
+
   pub async fn enqueue_generic_download(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> {
     let _record_id = self.insert(
       FirehoseEvent::GenericDownloadStarted,

crates/lib/mysql_queries/src/payloads/generic_inference_args.rs

@@ -1,8 +1,5 @@
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use errors::AnyhowResult;
-use tokens::files::media_upload::MediaUploadToken;
-use tokens::tokens::tts_models::TtsModelToken;
-use tokens::voice_conversion::model::VoiceConversionModelToken;
 
 /// Used to encode extra state for the `generic_inference_jobs` table.
 /// This should act somewhat like a serialized protobuf stored inside a record.
@@ -23,6 +20,10 @@ pub struct GenericInferenceArgs {
 /// Do not change the values.
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
 pub enum InferenceCategoryAbbreviated {
+  #[serde(rename = "la")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(alias = "lipsync_animation")]
+  LipsyncAnimation,
+
   #[serde(rename = "tts")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
   #[serde(alias = "text_to_speech")]
   TextToSpeech,
@@ -44,6 +45,17 @@ pub enum FundamentalFrequencyMethodForJob {
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub enum PolymorphicInferenceArgs {
+  /// Lipsync Animation (Short name to save  space when serializing)
+  /// This is SadTalker, not Wav2Lip.
+  La {
+    #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    maybe_audio_media_upload_token: Option<String>,
+
+    #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    maybe_image_media_upload_token: Option<String>,
+  },
   /// Text to speech. (Short name to save space when serializing.)
   Tts {
     // No arguments yet.
@@ -88,6 +100,7 @@ impl GenericInferenceArgs {
 impl InferenceCategoryAbbreviated {
   pub fn from_inference_category(category: InferenceCategory) -> Self {
     match category {
+      InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation,
       InferenceCategory::TextToSpeech => Self::TextToSpeech,
       InferenceCategory::VoiceConversion => Self::VoiceConversion,
     }
@@ -95,6 +108,7 @@ impl InferenceCategoryAbbreviated {
 
   pub fn to_inference_category(self) -> InferenceCategory {
     match self {
+      Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation,
       Self::TextToSpeech => InferenceCategory::TextToSpeech,
       Self::VoiceConversion => InferenceCategory::VoiceConversion,
     }
@@ -104,10 +118,25 @@ impl InferenceCategoryAbbreviated {
 #[cfg(test)]
 mod tests {
   use crate::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
-  use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
-  use tokens::files::media_upload::MediaUploadToken;
-  use tokens::tokens::tts_models::TtsModelToken;
-  use tokens::voice_conversion::model::VoiceConversionModelToken;
+
+  #[test]
+  fn typical_lipsync_animation_args_serialize() {
+    let args = GenericInferenceArgs {
+      inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation),
+      args: Some(PolymorphicInferenceArgs::La {
+        maybe_audio_media_upload_token: Some("foo".to_string()),
+        maybe_image_media_upload_token: Some("bar".to_string()),
+      }),
+    };
+
+    let json = serde_json::ser::to_string(&args).unwrap();
+
+    // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating.
+    assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":"foo","i":"bar"}}}"#.to_string());
+
+    // NB: Make sure we don't overflow the DB field capacity (TEXT column).
+    assert!(json.len() < 1000);
+  }
 
   #[test]
   fn typical_tts_args_serialize() {

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -120,6 +120,7 @@ WHERE jobs.token = ?
   };
 
   let maybe_model_title = match record.inference_category {
+    InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
     InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
     InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
   };
@@ -127,6 +128,7 @@ WHERE jobs.token = ?
   // NB: A bit of a hack. We store TTS results with a full path.
   // Going forward, all other record types will store a hash.
   let (bucket_path_is_hash, maybe_public_bucket_hash) = match record.inference_category {
+    InferenceCategory::LipsyncAnimation => (true, Some("todo")), // TODO - these values are wrong
     InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
     InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
   };

crates/service/web/storyteller_web/src/configs/plans/plan.rs

@@ -42,6 +42,10 @@ pub struct Plan {
     /// Whether this plan does not exist in Stripe and only exists as defined by this server.
     is_synthetic_plan: bool,
 
+    // ========== Features for lipsync (SadTalker, not Wav2Lip) ==========
+
+    lipsync_requires_frontend_keepalive: bool,
+
     // ========== Features for TTS ==========
 
     tts_base_priority_level: u8,
@@ -83,6 +87,7 @@ impl Plan {
             stripe_price_id: builder.stripe_price_id.clone(),
             is_development_plan: builder.is_development_plan,
             is_synthetic_plan: builder.is_synthetic_plan,
+            lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive,
             tts_base_priority_level: builder.tts_base_priority_level,
             tts_max_duration: builder.tts_max_duration,
             tts_max_character_length: builder.tts_max_character_length,
@@ -133,6 +138,10 @@ impl Plan {
         self.is_synthetic_plan
     }
 
+    pub fn lipsync_requires_frontend_keepalive(&self) -> bool {
+        self.lipsync_requires_frontend_keepalive
+    }
+
     pub fn tts_base_priority_level(&self) -> u8 {
         self.tts_base_priority_level
     }
@@ -169,6 +178,10 @@ pub struct PlanBuilder {
     is_development_plan: bool,
     is_synthetic_plan: bool,
 
+    // ========== Features for lipsync (SadTalker, not Wav2Lip) ==========
+
+    lipsync_requires_frontend_keepalive: bool,
+
     // ========== Features for TTS ==========
 
     tts_base_priority_level: u8,
@@ -212,6 +225,9 @@ impl PlanBuilder {
             is_development_plan: false,
             is_synthetic_plan: false,
 
+            // Lipsync (SadTalker, not Wav2Lip)
+            lipsync_requires_frontend_keepalive: true,
+
             // TTS
             tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL,
             tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS),
@@ -272,6 +288,11 @@ impl PlanBuilder {
         self
     }
 
+    pub fn lipsync_requires_frontend_keepalive(mut self, value: bool) -> Self {
+        self.lipsync_requires_frontend_keepalive = value;
+        self
+    }
+
     pub fn tts_base_priority_level(mut self, value: u8) -> Self {
         self.tts_base_priority_level = value;
         self

crates/service/web/storyteller_web/src/configs/plans/plan_list.rs

@@ -15,6 +15,7 @@ pub static FREE_LOGGED_OUT_PLAN : Lazy<Plan> = Lazy::new(|| {
         .tts_max_character_length(1024)
         .web_vc_base_priority_level(FAKEYOU_ANONYMOUS_PRIORITY_LEVEL)
         .web_vc_requires_frontend_keepalive(true)
+        .lipsync_requires_frontend_keepalive(true)
         .build()
 });
 
@@ -28,6 +29,7 @@ pub static FREE_LOGGED_OUT_FIRST_TRY_PLAN : Lazy<Plan> = Lazy::new(|| {
         .tts_max_character_length(1024)
         .web_vc_base_priority_level(FAKEYOU_LOGGED_IN_PRIORITY_LEVEL) // NB: Same as logged-in free users.
         .web_vc_requires_frontend_keepalive(true)
+        .lipsync_requires_frontend_keepalive(true)
         .build()
 });
 
@@ -41,6 +43,7 @@ pub static FREE_LOGGED_IN_PLAN : Lazy<Plan> = Lazy::new(|| {
         .tts_max_character_length(1024)
         .web_vc_base_priority_level(FAKEYOU_LOGGED_IN_PRIORITY_LEVEL)
         .web_vc_requires_frontend_keepalive(true)
+        .lipsync_requires_frontend_keepalive(true)
         .build()
 });
 
@@ -56,6 +59,7 @@ pub static LOYALTY_PLANS : Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(30)
         .web_vc_base_priority_level(2)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans
@@ -77,6 +81,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_character_length(2048)
         .tts_max_duration_seconds(30)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("development_fakeyou_pro")
@@ -89,6 +94,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_character_length(2048)
         .tts_max_duration_seconds(60 * 2)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("development_fakeyou_elite")
@@ -101,6 +107,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_character_length(2048)
         .tts_max_duration_seconds(60 * 5)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans
@@ -122,6 +129,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(30)
         .web_vc_base_priority_level(20)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("fakeyou_pro")
@@ -134,6 +142,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(60 * 2)
         .web_vc_base_priority_level(30)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     plans.insert(PlanBuilder::new("fakeyou_elite")
@@ -146,6 +155,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_max_duration_seconds(60 * 5)
         .web_vc_base_priority_level(40)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .build());
 
     // ========== Spanish plans ==========
@@ -158,6 +168,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy<HashSet<Plan>> = Lazy::new(|| {
         .tts_base_priority_level(10)
         .web_vc_base_priority_level(10)
         .web_vc_requires_frontend_keepalive(false)
+        .lipsync_requires_frontend_keepalive(false)
         .tts_max_duration_seconds(30)
         .build());
 
@@ -405,6 +416,7 @@ mod test {
     fn test_assert_web_vc_keepalive_forced() {
         FREE_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| {
             assert!(plan.web_vc_requires_frontend_keepalive());
+            assert!(plan.lipsync_requires_frontend_keepalive());
         });
     }
 
@@ -412,9 +424,11 @@ mod test {
     fn test_assert_web_vc_no_keepalive() {
         LOYALTY_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| {
             assert!(!plan.web_vc_requires_frontend_keepalive());
+            assert!(!plan.lipsync_requires_frontend_keepalive());
         });
         PRODUCTION_PREMIUM_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| {
             assert!(!plan.web_vc_requires_frontend_keepalive());
+            assert!(!plan.lipsync_requires_frontend_keepalive());
         });
     }
 

crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs

@@ -0,0 +1,300 @@
+// NB: Incrementally getting rid of build warnings...
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+use actix_web::error::ResponseError;
+use actix_web::http::StatusCode;
+use actix_web::{web, HttpResponse, HttpRequest};
+use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
+use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie;
+use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
+use crate::server_state::ServerState;
+use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::common::visibility::Visibility;
+use http_server_common::request::get_request_header_optional::get_request_header_optional;
+use http_server_common::request::get_request_ip::get_request_ip;
+use log::{info, warn};
+use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job};
+use std::sync::Arc;
+use tokens::files::media_upload::MediaUploadToken;
+use tokens::jobs::inference::InferenceJobToken;
+use tokens::users::user::UserToken;
+use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL;
+
+/// Debug requests can get routed to special "debug-only" workers, which can
+/// be used to trial new code, run debugging, etc.
+const DEBUG_HEADER_NAME : &'static str = "enable-debug-mode";
+
+/// The routing tag header can send workloads to particular k8s hosts.
+/// This is useful for catching the live logs or intercepting the job.
+const ROUTING_TAG_HEADER_NAME : &'static str = "routing-tag";
+
+#[derive(Deserialize)]
+pub struct EnqueueLipsyncAnimationRequest {
+  uuid_idempotency_token: String,
+
+  // TODO: Eventually accept result tokens for audio.
+  source_image_media_upload_token: MediaUploadToken,
+  source_audio_media_upload_token: MediaUploadToken,
+
+  creator_set_visibility: Option<Visibility>,
+  is_storyteller_demo: Option<bool>,
+
+  /// SadTalker: parameter to animate the full body.
+  animate_full_body: Option<bool>,
+}
+
+#[derive(Serialize)]
+pub struct EnqueueLipsyncAnimationSuccessResponse {
+  pub success: bool,
+  pub inference_job_token: InferenceJobToken,
+}
+
+#[derive(Debug)]
+pub enum EnqueueLipsyncAnimationError {
+  BadInput(String),
+  NotAuthorized,
+  ServerError,
+  RateLimited,
+}
+
+impl ResponseError for EnqueueLipsyncAnimationError {
+  fn status_code(&self) -> StatusCode {
+    match *self {
+      EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST,
+      EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED,
+      EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
+      EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
+    }
+  }
+
+  fn error_response(&self) -> HttpResponse {
+    let error_reason = match self {
+      EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(),
+      EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(),
+      EnqueueLipsyncAnimationError::ServerError => "server error".to_string(),
+      EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(),
+    };
+
+    to_simple_json_error(&error_reason, self.status_code())
+  }
+}
+
+// NB: Not using derive_more::Display since Clion doesn't understand it.
+impl std::fmt::Display for EnqueueLipsyncAnimationError {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "{:?}", self)
+  }
+}
+
+pub async fn enqueue_lipsync_animation_handler(
+  http_request: HttpRequest,
+  request: web::Json<EnqueueLipsyncAnimationRequest>,
+  server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, EnqueueLipsyncAnimationError>
+{
+  let mut maybe_user_token : Option<UserToken> = None;
+  let mut priority_level ;
+  let disable_rate_limiter = false; // NB: Careful!
+
+  let mut mysql_connection = server_state.mysql_pool
+      .acquire()
+      .await
+      .map_err(|err| {
+        warn!("MySql pool error: {:?}", err);
+        EnqueueLipsyncAnimationError::ServerError
+      })?;
+
+  // ==================== USER SESSION ==================== //
+
+  let maybe_user_session = server_state
+      .session_checker
+      .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection)
+      .await
+      .map_err(|e| {
+        warn!("Session checker error: {:?}", e);
+        EnqueueLipsyncAnimationError::ServerError
+      })?;
+
+  if let Some(user_session) = maybe_user_session.as_ref() {
+    maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token));
+  }
+
+  // TODO: Plan should handle "first anonymous use" and "investor" cases.
+  let plan = get_correct_plan_for_session(
+    server_state.server_environment,
+    maybe_user_session.as_ref());
+
+  // TODO: Separate priority for animation.
+  priority_level = plan.web_vc_base_priority_level();
+
+  // ==================== INVESTOR PRIORITY ==================== //
+
+  // TODO/TEMP: Give investors even more priority
+  let mut is_investor = false;
+
+  {
+    // TODO/TEMP: The storyteller.io website's AJAX calls will set this.
+    //  This is just for the YCombinator demo.
+    match request.is_storyteller_demo {
+      Some(true) => {
+        is_investor = true;
+      },
+      _ => {},
+    };
+
+    // TODO/TEMP: The storyteller.io website will redirect and establish this cookie.
+    //  This is just for the YCombinator demo.
+    if request_has_demo_cookie(&http_request) {
+      is_investor = true;
+    }
+
+    if is_investor {
+      priority_level = FAKEYOU_INVESTOR_PRIORITY_LEVEL;
+    }
+  }
+
+  // ==================== DEBUG MODE + ROUTING TAG ==================== //
+
+  let is_debug_request =
+      get_request_header_optional(&http_request, DEBUG_HEADER_NAME)
+          .is_some();
+
+  let maybe_routing_tag=
+      get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME)
+          .map(|routing_tag| routing_tag.trim().to_string());
+
+  // ==================== RATE LIMIT ==================== //
+
+  if !disable_rate_limiter {
+    let mut rate_limiter = match maybe_user_session {
+      None => &server_state.redis_rate_limiters.logged_out,
+      Some(ref user) => {
+        if user.role.is_banned {
+          return Err(EnqueueLipsyncAnimationError::NotAuthorized);
+        }
+        &server_state.redis_rate_limiters.logged_in
+      },
+    };
+
+    // TODO/TEMP
+    if is_investor {
+      rate_limiter = &server_state.redis_rate_limiters.logged_in;
+    }
+
+    if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
+      return Err(EnqueueLipsyncAnimationError::RateLimited);
+    }
+  }
+
+  // ==================== LOOK UP MODEL INFO ==================== //
+
+  // TODO(bt): CHECK DATABASE!
+  let audio_media_token = request.source_audio_media_upload_token.clone();
+  let image_media_token = request.source_image_media_upload_token.clone();
+
+  // ==================== CHECK AND ENQUEUE ANIMATION ==================== //
+
+  //let mut redis = server_state.redis_pool
+  //    .get()
+  //    .map_err(|e| {
+  //      error!("redis error: {:?}", e);
+  //      EnqueueLipsyncAnimationError::ServerError
+  //    })?;
+
+  //let redis_count_key = RedisKeys::web_vc_model_usage_count(model_token.as_str());
+
+  //redis.incr(&redis_count_key, 1)
+  //    .map_err(|e| {
+  //      warn!("redis error: {:?}", e);
+  //      EnqueueLipsyncAnimationError::ServerError
+  //    })?;
+
+  let ip_address = get_request_ip(&http_request);
+
+  let maybe_user_preferred_visibility : Option<Visibility> = maybe_user_session
+      .as_ref()
+      .map(|user_session| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc
+
+  let set_visibility = request.creator_set_visibility
+      .or(maybe_user_preferred_visibility)
+      .unwrap_or(Visibility::Public);
+
+
+  info!("Creating lipsync animation job record...");
+
+  // TODO: Cleanup.
+  let maybe_args = Some(PolymorphicInferenceArgs::La {
+    maybe_audio_media_upload_token: Some(audio_media_token.to_string()),
+    maybe_image_media_upload_token: Some(image_media_token.to_string()),
+  });
+
+  // TODO: SadTalker arguments.
+  //let set_args = request.auto_predict_f0.is_some()
+  //    || request.transpose.is_some()
+  //    || request.override_f0_method.is_some();
+
+  //if set_args {
+  //  maybe_args = Some(PolymorphicInferenceArgs::Vc {
+  //    auto_predict_f0: request.auto_predict_f0,
+  //    override_f0_method: request.override_f0_method.map(|f| match f {
+  //      FundamentalFrequencyMethod::Crepe => FundamentalFrequencyMethodForJob::Crepe,
+  //      FundamentalFrequencyMethod::Dio => FundamentalFrequencyMethodForJob::Dio,
+  //      FundamentalFrequencyMethod::Harvest => FundamentalFrequencyMethodForJob::Harvest,
+  //    }),
+  //    transpose: request.transpose,
+  //  });
+  //}
+
+  let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
+    uuid_idempotency_token: &request.uuid_idempotency_token,
+    inference_category: InferenceCategory::LipsyncAnimation,
+    maybe_model_type: None, // NB: Model is static during inference
+    maybe_model_token: None, // NB: Model is static during inference
+    maybe_input_source_token: None, // TODO: Introduce a second foreign key ?
+    maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ?
+    maybe_raw_inference_text: None, // No text
+    maybe_inference_args: Some(GenericInferenceArgs {
+      inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation),
+      args: maybe_args,
+    }),
+    maybe_creator_user_token: maybe_user_token.as_ref(),
+    creator_ip_address: &ip_address,
+    creator_set_visibility: set_visibility,
+    priority_level,
+    requires_keepalive: plan.lipsync_requires_frontend_keepalive(),
+    is_debug_request,
+    maybe_routing_tag: maybe_routing_tag.as_deref(),
+    mysql_pool: &server_state.mysql_pool,
+  }).await;
+
+  let job_token = match query_result {
+    Ok((job_token, _id)) => job_token,
+    Err(err) => {
+      warn!("New generic inference job creation DB error: {:?}", err);
+      return Err(EnqueueLipsyncAnimationError::ServerError);
+    }
+  };
+
+  server_state.firehose_publisher.enqueue_lipsync_animation(
+    maybe_user_token.as_ref(),
+    &job_token)
+      .await
+      .map_err(|e| {
+        warn!("error publishing event: {:?}", e);
+        EnqueueLipsyncAnimationError::ServerError
+      })?;
+
+  let response = EnqueueLipsyncAnimationSuccessResponse {
+    success: true,
+    inference_job_token: job_token,
+  };
+
+  let body = serde_json::to_string(&response)
+      .map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?;
+
+  Ok(HttpResponse::Ok()
+      .content_type("application/json")
+      .body(body))
+}

crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs

@@ -0,0 +1 @@
+pub mod enqueue_lipsync_animation;

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs

@@ -216,6 +216,10 @@ pub async fn get_inference_job_status_handler(
     },
     maybe_result: record.maybe_result_details.map(|result_details| {
       let public_bucket_media_path = match inference_category {
+        InferenceCategory::LipsyncAnimation => {
+          // TODO
+          "TODO".to_string()
+        }
         InferenceCategory::TextToSpeech => {
           // NB: TTS results receive the legacy treatment where their table only reports the full bucket path
           result_details.public_bucket_location_or_hash

crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/handle_upload.rs

@@ -186,6 +186,10 @@ pub async fn handle_upload(
       "audio/opus" /* .opus */ => Some(MediaUploadType::Audio),
       "audio/x-flac" /* .flac */ => Some(MediaUploadType::Audio),
       "audio/x-wav" /* .wav */ => Some(MediaUploadType::Audio),
+      // Image
+      "image/jpeg" /* .jpg */ => Some(MediaUploadType::Image),
+      "image/png" /* .png */ => Some(MediaUploadType::Image),
+      "image/webp" /* .webp */ => Some(MediaUploadType::Image),
       // Video
       "video/mp4" /* .mp4 */ => Some(MediaUploadType::Video),
       "video/quicktime" /* .mov */ => Some(MediaUploadType::Video),
@@ -193,7 +197,7 @@ pub async fn handle_upload(
       _ => None,
     };
 
-    let do_decode = match mimetype {
+    let do_audio_decode = match mimetype {
       "audio/opus" => {
         // TODO/FIXME(bt, 2023-05-19): Symphonia is currently broken for Firefox's opus.
         //  We're on an off-master branch that may resolve the problem in the future, but for now
@@ -219,10 +223,14 @@ pub async fn handle_upload(
         //    right: `Ebml`: EBML element type must be checked before calling this function', /Users/bt/.cargo/git/checkouts/symphonia-8fbe6c90fc095688/e1a7009/symphonia-format-mkv/src/ebml.rs:335:9
         false
       }
+      // Also, don't decode images
+      "image/jpeg" => false,
+      "image/png" => false,
+      "image/webp" => false,
       _ => true,
     };
 
-    if do_decode && media_upload_type.is_some() {
+    if do_audio_decode && media_upload_type.is_some() {
       let basic_info = decode_basic_audio_bytes_info(
         bytes.as_ref(),
         Some(mimetype),

crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/mod.rs

@@ -1,4 +1,5 @@
 pub mod common;
 pub mod list_user_media_uploads_of_type;
 pub mod upload_audio;
+pub mod upload_image;
 pub mod upload_media;

crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_image.rs

@@ -0,0 +1,50 @@
+use actix_multipart::Multipart;
+use actix_web::{HttpRequest, HttpResponse, web};
+use crate::http_server::endpoints::media_uploads::common::handle_upload::handle_upload;
+use crate::http_server::endpoints::media_uploads::common::upload_error::UploadError;
+use crate::server_state::ServerState;
+use once_cell::sync::Lazy;
+use std::collections::HashSet;
+use std::sync::Arc;
+use tokens::files::media_upload::MediaUploadToken;
+
+#[derive(Serialize)]
+pub struct UploadAudioSuccessResponse {
+  pub success: bool,
+  pub upload_token: MediaUploadToken,
+}
+
+static ALLOWED_MIME_TYPES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
+  HashSet::from([
+    "image/jpeg",
+    "image/png",
+    "image/webp",
+  ])
+});
+
+pub async fn upload_image_handler(
+  http_request: HttpRequest,
+  server_state: web::Data<Arc<ServerState>>,
+  mut multipart_payload: Multipart,
+) -> Result<HttpResponse, UploadError> {
+
+  let response = handle_upload(
+    &http_request,
+    &server_state,
+    multipart_payload,
+    &ALLOWED_MIME_TYPES).await?;
+
+  let media_upload_token = response.to_media_token();
+
+  let response = UploadAudioSuccessResponse {
+    success: true,
+    upload_token: media_upload_token,
+  };
+
+  let body = serde_json::to_string(&response)
+      .map_err(|e| UploadError::ServerError)?;
+
+  return Ok(HttpResponse::Ok()
+      .content_type("application/json")
+      .body(body));
+}

crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs

@@ -1,3 +1,4 @@
+pub mod animation;
 pub mod api_tokens;
 pub mod categories;
 pub mod comments;