fal errors handled

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs

@@ -1,11 +1,8 @@
 use std::fmt;
 use std::sync::Arc;
 
-use crate::http_server::endpoints::webhooks::handle_image_payload::handle_image_payload;
-use crate::http_server::endpoints::webhooks::handle_images_payload::handle_images_payload;
-use crate::http_server::endpoints::webhooks::handle_model_glb_payload::handle_model_glb_payload;
-use crate::http_server::endpoints::webhooks::handle_model_mesh_payload::handle_model_mesh_payload;
-use crate::http_server::endpoints::webhooks::handle_video_payload::handle_video_payload;
+use crate::http_server::endpoints::webhooks::process_failure::handle_failed_fal_webhook::handle_failed_fal_webhook;
+use crate::http_server::endpoints::webhooks::process_success::handle_successful_fal_webhook::handle_sucessful_fal_webhook;
 use crate::state::server_state::ServerState;
 use actix_web::error::ResponseError;
 use actix_web::http::StatusCode;
@@ -15,24 +12,11 @@ use anyhow::Error;
 use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
 use log::{error, info, warn};
-use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::get_inference_job_by_fal_id;
-use mysql_queries::queries::generic_inference::fal::mark_fal_generic_inference_job_successfully_done::{mark_fal_generic_inference_job_successfully_done, MarkJobArgs};
 use pager::notification::notification_details_builder::NotificationDetailsBuilder;
 use pager::notification::notification_urgency::NotificationUrgency;
 use serde_json::Value;
 use utoipa::ToSchema;
 
-// 1. tauri --> hit endpoint to enqueue
-//
-// 2. webhook
-//  - upload media file
-//  - update job record with media token + status
-//
-// 3. tauri --> storyteller jobs endpoint polls
-//  - alert frontend of completion
-//
-// 4. javascript polls tauri  (backend removal)
-
 // TODO(bt, 2025-06-03): Handle webhook crypto authentication
 #[derive(Debug, Deserialize, ToSchema)]
 pub struct FalWebhookRequest {
@@ -123,8 +107,10 @@ pub async fn fal_webhook_handler(
       .as_deref()
       .ok_or_else(|| FalWebhookError::BadInput("Missing request_id".to_string()))?;
 
-  info!("FAL webhook request_id: {}", request_id);
+  info!("FAL webhook request_id: {} (status: {:?})", request_id, request.status);
 
+  // TODO(bt): Longer term, we should just use `fal_client` to parse webhooks and add lots of integration tests
+  //  across dozens of real messages.
   let payload = request.payload
       .as_ref()
       .ok_or_else(|| {
@@ -132,7 +118,19 @@ pub async fn fal_webhook_handler(
         FalWebhookError::BadInput("Missing payload".to_string())
       })?;
 
-  let result = process_fal_webhook(&server_state, request_id, payload).await;
+  let result = match request.status {
+    FalWebhookStatus::Ok => {
+      handle_sucessful_fal_webhook(&server_state, request_id, payload).await
+    }
+    FalWebhookStatus::Error => {
+      handle_failed_fal_webhook(
+        &server_state,
+        request_id,
+        payload,
+        request.error.as_deref(),
+      ).await
+    }
+  };
 
   if let Err(ref err) = result {
     error!("FAL webhook error for request_id {}: {:?}", request_id, err);
@@ -156,72 +154,3 @@ pub async fn fal_webhook_handler(
 
   result
 }
-
-async fn process_fal_webhook(
-  server_state: &ServerState,
-  request_id: &str,
-  payload: &Value,
-) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
-
-  let db_result = get_inference_job_by_fal_id(
-    request_id,
-    &server_state.mysql_pool,
-  ).await;
-
-  let job = match db_result {
-    Ok(Some(record)) => record,
-    Ok(None) => {
-      warn!("Could not find job record by fal request_id: {}", request_id);
-      return Err(FalWebhookError::NotFound)
-    },
-    Err(err) => {
-      warn!("Error querying job record for request_id {}: {:?}", request_id, err);
-      return Err(FalWebhookError::ServerError);
-    }
-  };
-
-  info!("Fal webhook job record for request_id {}: {:?}", request_id, job);
-
-  let mut maybe_media_token = None;
-  let mut maybe_batch_token = None;
-
-  if let Some(payload_obj) = payload.as_object() {
-    if payload_obj.contains_key("image") {
-      info!("Handling image payload for request_id {} / job {:?}", request_id, job.job_token);
-      let token = handle_image_payload(payload_obj, &job, server_state).await?;
-      maybe_media_token = Some(token);
-    } else if payload_obj.contains_key("images") {
-      (maybe_media_token, maybe_batch_token) = handle_images_payload(payload_obj, &job, server_state).await?;
-    } else if payload_obj.contains_key("video") {
-      info!("Handling video payload for request_id {} / job {:?}", request_id, job.job_token);
-      let token = handle_video_payload(payload_obj, &job, server_state).await?;
-      maybe_media_token = Some(token);
-    } else if payload_obj.contains_key("model_glb") {
-      info!("Handling model_glb payload for request_id {} / job {:?}", request_id, job.job_token);
-      let token = handle_model_glb_payload(payload_obj, &job, server_state).await?;
-      maybe_media_token = Some(token);
-    } else if payload_obj.contains_key("model_mesh") {
-      info!("Handling model_mesh payload for request_id {} / job {:?}", request_id, job.job_token);
-      let token = handle_model_mesh_payload(payload_obj, &job, server_state).await?;
-      maybe_media_token = Some(token);
-    }
-  }
-
-  if let Some(media_token) = maybe_media_token {
-    info!("Media file token for request_id {}: {:?}", request_id, media_token);
-    mark_fal_generic_inference_job_successfully_done(MarkJobArgs {
-      job_token: &job.job_token,
-      media_file_token: &media_token,
-      maybe_batch_token: maybe_batch_token.as_ref(),
-      mysql_executor: &server_state.mysql_pool,
-      phantom: Default::default(),
-    }).await.map_err(|err| {
-      warn!("Error marking job as successfully done for request_id {}: {:?}", request_id, err);
-      FalWebhookError::ServerError
-    })?;
-  } else {
-    warn!("No media token found in payload for request_id {} / job {:?}", request_id, job.job_token);
-  }
-
-  Ok(SimpleGenericJsonSuccess::wrapped(true))
-}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs

@@ -1,6 +1,3 @@
 pub mod fal_webhook_handler;
-pub mod handle_image_payload;
-pub mod handle_images_payload;
-pub mod handle_model_mesh_payload;
-pub mod handle_video_payload;
-pub mod handle_model_glb_payload;
+pub (super) mod process_failure;
+pub (super) mod process_success;

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/fal_error_detail.rs

@@ -0,0 +1,142 @@
+use serde_json::Value;
+
+/// A single error detail item from a FAL webhook error payload.
+///
+/// See: https://fal.ai/docs/documentation/model-apis/errors
+#[derive(Debug, Clone)]
+pub struct FalErrorDetail {
+  /// Human-readable error message. Should not be parsed for logic.
+  pub msg: Option<String>,
+
+  /// Machine-readable error type (e.g. "image_too_small", "image_too_large").
+  pub error_type: Option<String>,
+}
+
+/// Attempt to parse the "detail" array from a FAL webhook payload.
+///
+/// The payload may contain a "detail" key with an array of error objects,
+/// each having optional "msg" and "type" fields.
+///
+/// Returns an empty vec if the "detail" key is missing or not an array.
+pub fn parse_fal_error_details(payload: &Value) -> Vec<FalErrorDetail> {
+  let detail_array = match payload.get("detail").and_then(|v| v.as_array()) {
+    Some(arr) => arr,
+    None => return Vec::new(),
+  };
+
+  detail_array
+    .iter()
+    .map(|item| FalErrorDetail {
+      msg: item.get("msg").and_then(|v| v.as_str()).map(|s| s.to_string()),
+      error_type: item.get("type").and_then(|v| v.as_str()).map(|s| s.to_string()),
+    })
+    .collect()
+}
+
+/// Build a human-readable summary of the error details for logging and alerting.
+pub fn summarize_fal_error_details(details: &[FalErrorDetail]) -> String {
+  if details.is_empty() {
+    return "No error details provided.".to_string();
+  }
+
+  details
+    .iter()
+    .enumerate()
+    .map(|(i, detail)| {
+      let error_type = detail.error_type.as_deref().unwrap_or("unknown");
+      let msg = detail.msg.as_deref().unwrap_or("no message");
+      format!("[{}] type={}, msg={}", i + 1, error_type, msg)
+    })
+    .collect::<Vec<_>>()
+    .join("\n")
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use serde_json::json;
+
+  #[test]
+  fn parse_typical_error_detail() {
+    let payload = json!({
+      "detail": [
+        {
+          "loc": ["body", "image_url"],
+          "msg": "Image too small",
+          "type": "image_too_small",
+          "url": "https://docs.fal.ai/errors/#image_too_small",
+          "ctx": { "min_height": 512, "min_width": 512 }
+        }
+      ]
+    });
+
+    let details = parse_fal_error_details(&payload);
+    assert_eq!(details.len(), 1);
+    assert_eq!(details[0].msg.as_deref(), Some("Image too small"));
+    assert_eq!(details[0].error_type.as_deref(), Some("image_too_small"));
+  }
+
+  #[test]
+  fn parse_multiple_error_details() {
+    let payload = json!({
+      "detail": [
+        { "msg": "First error", "type": "err_one" },
+        { "msg": "Second error", "type": "err_two" }
+      ]
+    });
+
+    let details = parse_fal_error_details(&payload);
+    assert_eq!(details.len(), 2);
+    assert_eq!(details[0].error_type.as_deref(), Some("err_one"));
+    assert_eq!(details[1].error_type.as_deref(), Some("err_two"));
+  }
+
+  #[test]
+  fn parse_missing_detail_field() {
+    let payload = json!({ "images": [] });
+    let details = parse_fal_error_details(&payload);
+    assert!(details.is_empty());
+  }
+
+  #[test]
+  fn parse_detail_not_array() {
+    let payload = json!({ "detail": "some string" });
+    let details = parse_fal_error_details(&payload);
+    assert!(details.is_empty());
+  }
+
+  #[test]
+  fn parse_detail_with_missing_fields() {
+    let payload = json!({
+      "detail": [
+        { "msg": "Only message" },
+        { "type": "only_type" },
+        {}
+      ]
+    });
+
+    let details = parse_fal_error_details(&payload);
+    assert_eq!(details.len(), 3);
+    assert_eq!(details[0].msg.as_deref(), Some("Only message"));
+    assert_eq!(details[0].error_type, None);
+    assert_eq!(details[1].msg, None);
+    assert_eq!(details[1].error_type.as_deref(), Some("only_type"));
+    assert_eq!(details[2].msg, None);
+    assert_eq!(details[2].error_type, None);
+  }
+
+  #[test]
+  fn summarize_empty() {
+    assert_eq!(summarize_fal_error_details(&[]), "No error details provided.");
+  }
+
+  #[test]
+  fn summarize_single() {
+    let details = vec![FalErrorDetail {
+      msg: Some("Image too small".to_string()),
+      error_type: Some("image_too_small".to_string()),
+    }];
+    let summary = summarize_fal_error_details(&details);
+    assert_eq!(summary, "[1] type=image_too_small, msg=Image too small");
+  }
+}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/handle_failed_fal_webhook.rs

@@ -0,0 +1,122 @@
+use actix_web::web::Json;
+use log::{error, info, warn};
+use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::get_inference_job_by_fal_id;
+use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs};
+use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
+use serde_json::Value;
+use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
+use crate::http_server::endpoints::webhooks::fal_webhook_handler::FalWebhookError;
+use crate::http_server::endpoints::webhooks::process_failure::fal_error_detail::{FalErrorDetail, parse_fal_error_details, summarize_fal_error_details};
+use crate::state::server_state::ServerState;
+
+/// Map FAL error type strings to frontend failure categories.
+///
+/// Returns `GenerationFailed` as the default if no error details match a known type.
+fn guess_failure_category(details: &[FalErrorDetail]) -> FrontendFailureCategory {
+  for detail in details {
+    if let Some(error_type) = &detail.error_type {
+      match error_type.as_str() {
+        "content_policy_violation" => return FrontendFailureCategory::RuleBansUserContent,
+        "face_detection_error" => return FrontendFailureCategory::FaceNotDetected,
+        // These all map to GenerationFailed for now, but are listed explicitly
+        // so we can refine them later.
+        "no_media_generated"
+        | "image_too_small"
+        | "image_too_large"
+        | "image_load_error"
+        | "file_download_error"
+        | "file_too_large" => return FrontendFailureCategory::GenerationFailed,
+        _ => {}
+      }
+    }
+  }
+  FrontendFailureCategory::GenerationFailed
+}
+
+/// Handle a FAL webhook with status ERROR.
+///
+/// Parses the error details from the payload, looks up the job, and marks it as failed.
+pub async fn handle_failed_fal_webhook(
+  server_state: &ServerState,
+  request_id: &str,
+  payload: &Value,
+  maybe_top_level_error: Option<&str>,
+) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
+
+  // Parse the "detail" array from the payload.
+  let error_details = parse_fal_error_details(payload);
+  let error_summary = summarize_fal_error_details(&error_details);
+
+  info!(
+    "FAL webhook ERROR for request_id {}: top_level_error={:?}, details=[{}]",
+    request_id,
+    maybe_top_level_error,
+    error_summary,
+  );
+
+  // Look up the job record.
+  let job = match get_inference_job_by_fal_id(request_id, &server_state.mysql_pool).await {
+    Ok(Some(record)) => record,
+    Ok(None) => {
+      warn!("Could not find job record by fal request_id: {}", request_id);
+      return Err(FalWebhookError::NotFound);
+    }
+    Err(err) => {
+      error!("Error querying job record for request_id {}: {:?}", request_id, err);
+      return Err(FalWebhookError::ServerError);
+    }
+  };
+
+  // Build a failure reason from the error details or the top-level error string.
+  let public_failure_reason = if !error_details.is_empty() {
+    error_details.iter()
+        .filter_map(|d| d.msg.as_deref())
+        .collect::<Vec<_>>()
+        .join("; ")
+  } else if let Some(top_level) = maybe_top_level_error {
+    top_level.to_string()
+  } else {
+    "Unknown FAL error".to_string()
+  };
+
+  let internal_failure_reason = format!(
+    "FAL request_id={}, top_level_error={:?}, details=[{}]",
+    request_id,
+    maybe_top_level_error,
+    error_summary,
+  );
+
+  let failure_category = guess_failure_category(&error_details);
+
+  info!(
+    "Marking job {} as failed for request_id {}. Category: {:?}, Reason: {}",
+    job.job_token.as_str(),
+    request_id,
+    failure_category,
+    public_failure_reason,
+  );
+
+  if let Err(err) = mark_job_failed_by_token(MarkJobFailedByTokenArgs {
+    pool: &server_state.mysql_pool,
+    job_token: &job.job_token,
+    maybe_public_failure_reason: Some(&public_failure_reason),
+    internal_debugging_failure_reason: &internal_failure_reason,
+    maybe_frontend_failure_category: Some(failure_category),
+  }).await {
+    error!(
+      "Error marking job {} as failed for request_id {}: {:?}",
+      job.job_token.as_str(),
+      request_id,
+      err,
+    );
+    return Err(FalWebhookError::ServerError);
+  }
+
+  info!(
+    "Job {} marked as failed for request_id {}.",
+    job.job_token.as_str(),
+    request_id,
+  );
+
+  Ok(SimpleGenericJsonSuccess::wrapped(true))
+}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/mod.rs

@@ -0,0 +1,2 @@
+pub mod fal_error_detail;
+pub mod handle_failed_fal_webhook;

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/handle_successful_fal_webhook.rs

@@ -0,0 +1,84 @@
+use actix_web::web::Json;
+use log::{info, warn};
+use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::get_inference_job_by_fal_id;
+use mysql_queries::queries::generic_inference::fal::mark_fal_generic_inference_job_successfully_done::{mark_fal_generic_inference_job_successfully_done, MarkJobArgs};
+use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
+use serde_json::Value;
+
+use crate::http_server::endpoints::webhooks::fal_webhook_handler::FalWebhookError;
+use crate::state::server_state::ServerState;
+
+use super::process_image_payload::process_image_payload;
+use super::process_images_payload::process_images_payload;
+use super::process_model_glb_payload::process_model_glb_payload;
+use super::process_model_mesh_payload::process_model_mesh_payload;
+use super::process_video_payload::process_video_payload;
+
+pub async fn handle_sucessful_fal_webhook(
+  server_state: &ServerState,
+  request_id: &str,
+  payload: &Value,
+) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
+
+  let db_result = get_inference_job_by_fal_id(
+    request_id,
+    &server_state.mysql_pool,
+  ).await;
+
+  let job = match db_result {
+    Ok(Some(record)) => record,
+    Ok(None) => {
+      warn!("Could not find job record by fal request_id: {}", request_id);
+      return Err(FalWebhookError::NotFound)
+    },
+    Err(err) => {
+      warn!("Error querying job record for request_id {}: {:?}", request_id, err);
+      return Err(FalWebhookError::ServerError);
+    }
+  };
+
+  info!("Fal webhook job record for request_id {}: {:?}", request_id, job);
+
+  let mut maybe_media_token = None;
+  let mut maybe_batch_token = None;
+
+  if let Some(payload_obj) = payload.as_object() {
+    if payload_obj.contains_key("image") {
+      info!("Handling image payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = process_image_payload(payload_obj, &job, server_state).await?;
+      maybe_media_token = Some(token);
+    } else if payload_obj.contains_key("images") {
+      (maybe_media_token, maybe_batch_token) = process_images_payload(payload_obj, &job, server_state).await?;
+    } else if payload_obj.contains_key("video") {
+      info!("Handling video payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = process_video_payload(payload_obj, &job, server_state).await?;
+      maybe_media_token = Some(token);
+    } else if payload_obj.contains_key("model_glb") {
+      info!("Handling model_glb payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = process_model_glb_payload(payload_obj, &job, server_state).await?;
+      maybe_media_token = Some(token);
+    } else if payload_obj.contains_key("model_mesh") {
+      info!("Handling model_mesh payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = process_model_mesh_payload(payload_obj, &job, server_state).await?;
+      maybe_media_token = Some(token);
+    }
+  }
+
+  if let Some(media_token) = maybe_media_token {
+    info!("Media file token for request_id {}: {:?}", request_id, media_token);
+    mark_fal_generic_inference_job_successfully_done(MarkJobArgs {
+      job_token: &job.job_token,
+      media_file_token: &media_token,
+      maybe_batch_token: maybe_batch_token.as_ref(),
+      mysql_executor: &server_state.mysql_pool,
+      phantom: Default::default(),
+    }).await.map_err(|err| {
+      warn!("Error marking job as successfully done for request_id {}: {:?}", request_id, err);
+      FalWebhookError::ServerError
+    })?;
+  } else {
+    warn!("No media token found in payload for request_id {} / job {:?}", request_id, job.job_token);
+  }
+
+  Ok(SimpleGenericJsonSuccess::wrapped(true))
+}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/mod.rs

@@ -0,0 +1,6 @@
+pub mod handle_successful_fal_webhook;
+mod process_image_payload;
+mod process_images_payload;
+mod process_model_glb_payload;
+mod process_model_mesh_payload;
+mod process_video_payload;

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_image_payload.rs

@@ -29,7 +29,7 @@ pub struct FalWebhookImage {
   pub url: Option<String>,
 }
 
-pub async fn handle_image_payload(
+pub async fn process_image_payload(
   payload: &Map<String, Value>,
   job: &FalJobDetails,
   server_state: &ServerState,

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_images_payload.rs

@@ -30,7 +30,7 @@ pub struct FalWebhookImage {
   pub url: Option<String>,
 }
 
-pub async fn handle_images_payload(
+pub async fn process_images_payload(
   payload: &Map<String, Value>,
   job: &FalJobDetails,
   server_state: &ServerState,

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_glb_payload.rs

@@ -63,7 +63,7 @@ pub struct FalMediaPayload {
   pub url: Option<String>,
 }
 
-pub async fn handle_model_glb_payload(
+pub async fn process_model_glb_payload(
   payload: &Map<String, Value>,
   job: &FalJobDetails,
   server_state: &ServerState,

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_model_mesh_payload.rs

@@ -25,7 +25,7 @@ pub struct FalWebhookModelMesh {
   pub url: Option<String>,
 }
 
-pub async fn handle_model_mesh_payload(
+pub async fn process_model_mesh_payload(
   payload: &Map<String, Value>,
   job: &FalJobDetails,
   server_state: &ServerState,

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/process_video_payload.rs

@@ -31,7 +31,7 @@ pub struct FalWebhookVideo {
   pub url: Option<String>,
 }
 
-pub async fn handle_video_payload(
+pub async fn process_video_payload(
   payload: &Map<String, Value>,
   job: &FalJobDetails,
   server_state: &ServerState,