handle audio results (#1772)

crates/api_clients/fal_client/src/webhook_payload/hydrated/hydrated_webhook_contents.rs

@@ -43,6 +43,9 @@ pub struct ExtractedContents {
   /// Parsed from `payload.video`.
   pub video: Option<VideoData>,
 
+  /// Parsed from `payload.audio` (e.g. Seed Audio 1.0 speech results).
+  pub audio: Option<AudioData>,
+
   /// Parsed from `payload.model_glb`.
   pub model_glb: Option<ModelGlbData>,
 
@@ -108,6 +111,20 @@ pub struct VideoData {
   pub file_size: Option<u64>,
 }
 
+/// Data under `payload.audio` (e.g. Seed Audio 1.0):
+#[derive(Debug, Deserialize)]
+pub struct AudioData {
+  pub url: Option<String>,
+  pub content_type: Option<String>,
+  pub file_name: Option<String>,
+  pub file_size: Option<u64>,
+  pub bitrate: Option<u64>,
+  pub channels: Option<u64>,
+  /// Duration in seconds.
+  pub duration: Option<f64>,
+  pub sample_rate: Option<u64>,
+}
+
 /// Data under `payload.model_glb` (there may be other sibling keys too).
 /// Also used for `payload.model_glb_pbr`, which has the same shape.
 #[derive(Debug, Deserialize)]

crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/extract_audio_payload.rs

@@ -0,0 +1,103 @@
+use serde_json::{Map, Value};
+
+use crate::webhook_payload::hydrated::hydrated_webhook_contents::AudioData;
+
+/// Extract and deserialize the `audio` key from a webhook success payload
+/// (e.g. Seed Audio 1.0 speech results).
+pub(crate) fn extract_audio(obj: &Map<String, Value>) -> Option<AudioData> {
+  let value = obj.get("audio")?;
+  serde_json::from_value(value.clone()).ok()
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::webhook_payload::hydrate_webhook_contents::hydrate_webhook_contents;
+  use crate::webhook_payload::hydrated::hydrated_webhook_contents::HydratedWebhookContents;
+  use crate::webhook_payload::raw::raw_webhook_payload::RawWebhookPayload;
+
+  fn load_test_webhook(filename: &str) -> RawWebhookPayload {
+    let path = format!("test_data/webhooks/{}", filename);
+    let json = std::fs::read_to_string(&path)
+      .unwrap_or_else(|e| panic!("Failed to read {}: {}", path, e));
+    serde_json::from_str(&json)
+      .unwrap_or_else(|e| panic!("Failed to parse {}: {}", path, e))
+  }
+
+  #[test]
+  fn audio_from_seed_audio_1p0_test_file() {
+    let webhook = load_test_webhook("success/seed_audio_1p0_audio_payload_1.json");
+    let result = hydrate_webhook_contents(&webhook);
+
+    let HydratedWebhookContents::Success(data) = result else {
+      panic!("Expected Success, got {:?}", result);
+    };
+
+    let contents = data.extracted_contents
+      .expect("extracted_contents should be Some");
+
+    let audio = contents.audio.expect("audio should be Some");
+    assert_eq!(audio.url.as_deref(), Some("https://v3b.fal.media/files/b/0aa1bc36/vOSCcxsbibkZIiXR9BIZn_speech.mp3"));
+    assert_eq!(audio.content_type.as_deref(), Some("audio/mpeg"));
+    assert_eq!(audio.file_name.as_deref(), Some("speech.mp3"));
+    assert_eq!(audio.file_size, Some(53612));
+    assert_eq!(audio.duration, Some(6.63995));
+    assert_eq!(audio.sample_rate, Some(24000));
+    assert!(audio.bitrate.is_none());
+    assert!(audio.channels.is_none());
+
+    // No other content types in this payload.
+    assert!(contents.image.is_none());
+    assert!(contents.images.is_none());
+    assert!(contents.video.is_none());
+    assert!(contents.model_glb.is_none());
+    assert!(contents.thumbnail.is_none());
+  }
+
+  #[test]
+  fn synthetic_audio_payload() {
+    let obj: serde_json::Map<String, serde_json::Value> = serde_json::from_str(r#"{
+      "audio": {
+        "url": "https://cdn.example.com/speech.mp3",
+        "content_type": "audio/mpeg",
+        "file_name": "speech.mp3",
+        "file_size": 12345,
+        "bitrate": 128000,
+        "channels": 2,
+        "duration": 3.5,
+        "sample_rate": 44100
+      }
+    }"#).unwrap();
+
+    let audio = extract_audio(&obj).expect("should extract audio");
+    assert_eq!(audio.url.as_deref(), Some("https://cdn.example.com/speech.mp3"));
+    assert_eq!(audio.content_type.as_deref(), Some("audio/mpeg"));
+    assert_eq!(audio.file_name.as_deref(), Some("speech.mp3"));
+    assert_eq!(audio.file_size, Some(12345));
+    assert_eq!(audio.bitrate, Some(128000));
+    assert_eq!(audio.channels, Some(2));
+    assert_eq!(audio.duration, Some(3.5));
+    assert_eq!(audio.sample_rate, Some(44100));
+  }
+
+  #[test]
+  fn audio_url_only() {
+    let obj: serde_json::Map<String, serde_json::Value> = serde_json::from_str(r#"{
+      "audio": {"url": "https://cdn.example.com/speech.wav"}
+    }"#).unwrap();
+
+    let audio = extract_audio(&obj).expect("should extract audio");
+    assert_eq!(audio.url.as_deref(), Some("https://cdn.example.com/speech.wav"));
+    assert!(audio.content_type.is_none());
+    assert!(audio.duration.is_none());
+  }
+
+  #[test]
+  fn missing_audio_key_returns_none() {
+    let obj: serde_json::Map<String, serde_json::Value> = serde_json::from_str(r#"{
+      "video": {"url": "https://example.com/video.mp4"}
+    }"#).unwrap();
+
+    assert!(extract_audio(&obj).is_none());
+  }
+}

crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/mod.rs

@@ -1,3 +1,4 @@
+pub mod extract_audio_payload;
 pub mod extract_image_payload;
 pub mod extract_images_payload;
 pub mod extract_video_payload;
@@ -18,8 +19,8 @@ use crate::webhook_payload::hydrated::hydrated_webhook_contents::ExtractedConten
 /// Try to extract known content keys from a success payload.
 ///
 /// Checks the payload (as a JSON object) for any of: `image`, `images`,
-/// `video`, `model_glb`, `model_glb_pbr`, `model_urls`, `model_mesh`,
-/// `model_obj`, `result_files`, `thumbnail`, `rendered_image`,
+/// `video`, `audio`, `model_glb`, `model_glb_pbr`, `model_urls`,
+/// `model_mesh`, `model_obj`, `result_files`, `thumbnail`, `rendered_image`,
 /// `preprocessed_image`. If at least one is found, returns
 /// `Some(ExtractedContents)` with copies of the relevant values. Multiple
 /// keys can be populated simultaneously. Returns `None` if the payload is
@@ -30,6 +31,7 @@ pub fn extract_contents_from_payload(payload: &Value) -> Option<ExtractedContent
   let image = extract_image_payload::extract_image(obj);
   let images = extract_images_payload::extract_images(obj);
   let video = extract_video_payload::extract_video(obj);
+  let audio = extract_audio_payload::extract_audio(obj);
   let model_glb = extract_model_glb_payload::extract_model_glb(obj);
   let model_glb_pbr = extract_model_glb_pbr_payload::extract_model_glb_pbr(obj);
   let model_urls = extract_model_urls_payload::extract_model_urls(obj);
@@ -40,7 +42,7 @@ pub fn extract_contents_from_payload(payload: &Value) -> Option<ExtractedContent
   let rendered_image = extract_rendered_image_payload::extract_rendered_image(obj);
   let preprocessed_image = extract_preprocessed_image_payload::extract_preprocessed_image(obj);
 
-  if image.is_none() && images.is_none() && video.is_none()
+  if image.is_none() && images.is_none() && video.is_none() && audio.is_none()
     && model_glb.is_none() && model_glb_pbr.is_none() && model_urls.is_none()
     && model_mesh.is_none() && model_obj.is_none()
     && result_files.is_none() && thumbnail.is_none() && rendered_image.is_none()
@@ -53,6 +55,7 @@ pub fn extract_contents_from_payload(payload: &Value) -> Option<ExtractedContent
     image,
     images,
     video,
+    audio,
     model_glb,
     model_glb_pbr,
     model_urls,

crates/api_clients/fal_client/test_data/webhooks/success/seed_audio_1p0_audio_payload_1.json

@@ -0,0 +1,18 @@
+{
+  "error": null,
+  "gateway_request_id": "019f4dc7-e98b-7f90-94c5-a8b131ff2660",
+  "payload": {
+    "audio": {
+      "bitrate": null,
+      "channels": null,
+      "content_type": "audio/mpeg",
+      "duration": 6.63995,
+      "file_name": "speech.mp3",
+      "file_size": 53612,
+      "sample_rate": 24000,
+      "url": "https://v3b.fal.media/files/b/0aa1bc36/vOSCcxsbibkZIiXR9BIZn_speech.mp3"
+    }
+  },
+  "request_id": "019f4dc7-e98b-7f90-94c5-a8b131ff2660",
+  "status": "OK"
+}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs

@@ -13,6 +13,7 @@ use sqlx::MySql;
 use crate::http_server::common_responses::common_web_error::CommonWebError;
 use crate::state::server_state::ServerState;
 
+use super::process_audio_payload::process_audio_payload;
 use super::process_image_payload::process_image_payload;
 use super::process_images_payload::process_images_payload;
 use super::process_model_glb_payload::process_model_glb_payload;
@@ -96,6 +97,14 @@ pub async fn handle_successful_fal_webhook(
       }
     }
 
+    if let Some(ref audio_data) = extracted.audio {
+      info!("Handling audio payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = process_audio_payload(audio_data, &job, server_state).await?;
+      if maybe_media_token.is_none() {
+        maybe_media_token = Some(token);
+      }
+    }
+
     // NB: GLBs take priority over `model_mesh`. Hunyuan 3D 2.1 sends both,
     // where `model_mesh` is a zip archive of the whole generation that we
     // skip in favor of the GLBs (standard + optional PBR variant). The

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/mod.rs

@@ -1,5 +1,6 @@
 pub mod handle_successful_fal_webhook;
 mod attach_cover_image;
+mod process_audio_payload;
 mod process_image_payload;
 mod process_images_payload;
 mod process_model_glb_payload;

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_audio_payload.rs

@@ -0,0 +1,132 @@
+use crate::http_server::common_responses::common_web_error::CommonWebError;
+use crate::http_server::endpoints::webhooks::fal::process_success::resolve_file_metadata::resolve_file_metadata;
+use crate::state::server_state::ServerState;
+use crate::util::http_download_url_to_bytes::http_download_url_to_bytes;
+use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
+use enums::by_table::media_files::media_file_class::MediaFileClass;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::common::generation_provider::GenerationProvider;
+use fal_client::webhook_payload::hydrated::hydrated_webhook_contents::AudioData;
+use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
+use log::{info, warn};
+use mysql_queries::queries::generic_inference::api_providers::fal::get_inference_job_by_fal_id::FalJobDetails;
+use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
+use tokens::tokens::media_files::MediaFileToken;
+
+const PREFIX : Option<&str> = Some("artcraft_");
+
+/// Process an `audio` payload (e.g. Seed Audio 1.0 speech results): download
+/// the file, upload it to the public bucket, and insert an audio media file
+/// record.
+pub async fn process_audio_payload(
+  audio_data: &AudioData,
+  job: &FalJobDetails,
+  server_state: &ServerState,
+) -> Result<MediaFileToken, CommonWebError> {
+  let audio_url = audio_data.url
+      .as_deref()
+      .ok_or_else(|| {
+        warn!("No `url` in audio payload");
+        CommonWebError::server_error_with_message("no `url` in audio payload")
+      })?;
+
+  // Download with a retry if the first attempt returns suspiciously few bytes.
+  let mut file_bytes = http_download_url_to_bytes(audio_url)
+      .await
+      .map_err(|err| {
+        warn!("Failed to download audio from {}: {:?}", audio_url, err);
+        CommonWebError::from_error(err)
+      })?;
+
+  if file_bytes.len() <= 10 {
+    warn!(
+      "Downloaded only {} bytes from {} — retrying once",
+      file_bytes.len(),
+      audio_url,
+    );
+    file_bytes = http_download_url_to_bytes(audio_url)
+        .await
+        .map_err(|err| {
+          warn!("Failed to download audio on retry from {}: {:?}", audio_url, err);
+          CommonWebError::from_error(err)
+        })?;
+  }
+
+  // Resolve mime type: magic bytes first, fal content_type as fallback.
+  let metadata = resolve_file_metadata(&file_bytes, audio_data.content_type.as_deref())
+      .ok_or_else(|| {
+        warn!(
+          "Could not determine file type for audio (bytes: {}, fal content_type: {:?})",
+          file_bytes.len(),
+          audio_data.content_type,
+        );
+        CommonWebError::server_error_with_message(
+          &format!("Could not determine file type for audio (bytes: {}, fal content_type: {:?})",
+            file_bytes.len(), audio_data.content_type))
+      })?;
+
+  let mime_type = metadata.mime_type.as_str();
+
+  info!("Mime type of audio: {}, source: {:?}", mime_type, metadata.source);
+
+  let media_file_type = MediaFileType::try_from_mime_type(mime_type)
+      .ok_or_else(|| {
+        warn!("Unsupported media file type: {}", mime_type);
+        CommonWebError::server_error_with_message(
+          &format!("Unsupported media file type: {}", mime_type))
+      })?;
+
+  let extension_with_period = metadata.file_extension.extension_with_period();
+
+  let file_size_bytes = file_bytes.len();
+  let file_hash = sha256_hash_bytes(&file_bytes)
+      .map_err(|err| {
+        warn!("Failed to hash audio bytes: {:?}", err);
+        CommonWebError::from_anyhow_error(err)
+      })?;
+
+  // Fal reports the duration in (fractional) seconds.
+  let maybe_duration_millis = audio_data.duration
+      .map(|seconds| (seconds * 1000.0) as u64);
+
+  let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension_with_period));
+
+  info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
+
+  server_state.public_bucket_client.upload_file_with_content_type_process(
+    public_upload_path.get_full_object_path_str(),
+    file_bytes.as_ref(),
+    &mime_type)
+      .await
+      .map_err(|err| {
+        warn!("Failed to upload audio to bucket: {:?}", err);
+        CommonWebError::from_anyhow_error(err)
+      })?;
+
+  let media_token = MediaFileInsertBuilder::new()
+      .checksum_sha2(&file_hash)
+      .creator_ip_address(&job.creator_ip_address)
+      .file_size_bytes(file_size_bytes as u64)
+      .maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref())
+      .maybe_creator_user(job.maybe_creator_user_token.as_ref())
+      .maybe_duration_millis(maybe_duration_millis)
+      .maybe_generation_provider(Some(GenerationProvider::Artcraft))
+      .maybe_prompt_token(job.maybe_prompt_token.as_ref())
+      .maybe_platform_type(job.maybe_platform_type)
+      .media_file_class(MediaFileClass::Audio)
+      .media_file_origin_category(MediaFileOriginCategory::Inference)
+      .media_file_type(media_file_type)
+      .mime_type(mime_type)
+      .public_bucket_directory_hash(&public_upload_path)
+      .insert_pool(&server_state.mysql_pool)
+      .await
+      .map_err(|err| {
+        warn!("Failed to insert audio media file record: {:?}", err);
+        CommonWebError::from_error(err)
+      })?;
+
+  info!("Audio media file uploaded with token: {}", media_token);
+
+  Ok(media_token)
+}