nit

crates/service/web/storyteller_web/src/docs/api_doc.rs

@@ -307,7 +307,6 @@ use artcraft_api_defs::moderation::wallets::moderator_create_wallet_for_user::*;
 use artcraft_api_defs::moderation::wallets::moderator_get_wallet::*;
 use artcraft_api_defs::web_referrals::log_web_referral::*;
 use crate::http_server::endpoints::web_referrals::log_web_referral_handler::*;
-use crate::http_server::endpoints::webhooks::fal_webhook_handler::*;
 use crate::http_server::endpoints::image_studio::update_gpt_image_job_status_handler::*;
 use crate::http_server::endpoints::voice_conversion::enqueue_seed_vc_inference_handler::*;
 use crate::http_server::endpoints::credits::get_session_credits_handler::*;
@@ -515,8 +514,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
     crate::http_server::endpoints::subscriptions::get_session_subscription_handler::get_session_subscription_handler,
     // Web Referrals
     crate::http_server::endpoints::web_referrals::log_web_referral_handler::log_web_referral_handler,
-    // Webhooks
-    crate::http_server::endpoints::webhooks::fal_webhook_handler::fal_webhook_handler,
     // Image Studio
     crate::http_server::endpoints::image_studio::update_gpt_image_job_status_handler::update_gpt_image_job_status_handler,
   ),

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs

@@ -2,7 +2,7 @@ use std::fmt;
 use std::sync::Arc;
 
 use crate::http_server::endpoints::webhooks::process_failure::handle_failed_fal_webhook::handle_failed_fal_webhook;
-use crate::http_server::endpoints::webhooks::process_success::handle_successful_fal_webhook::handle_sucessful_fal_webhook;
+use crate::http_server::endpoints::webhooks::process_success::handle_successful_fal_webhook::handle_successful_fal_webhook;
 use crate::state::server_state::ServerState;
 use actix_web::error::ResponseError;
 use actix_web::http::StatusCode;
@@ -10,37 +10,16 @@ use actix_web::web::Bytes;
 use actix_web::web::Json;
 use actix_web::{web, HttpRequest, HttpResponse};
 use anyhow::Error;
-use fal_client::webhook_api::payload::webhook_payload::{WebhookPayload, WebhookStatus};
+use fal_client::webhook_api::parse_webhook_inner_payload::parse_webhook_inner_payload;
+use fal_client::webhook_api::payload::webhook_inner_payload::WebhookInnerPayload;
+use fal_client::webhook_api::parse_webhook_payload::parse_webhook_payload;
 use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
 use log::{error, info, warn};
 use pager::notification::notification_details_builder::NotificationDetailsBuilder;
 use pager::notification::notification_urgency::NotificationUrgency;
-use serde_json::Value;
 use utoipa::ToSchema;
 
-// TODO(bt, 2025-06-03): Handle webhook crypto authentication
-#[derive(Debug, Deserialize, ToSchema)]
-pub struct FalWebhookRequest {
-  pub status: FalWebhookStatus,
-
-  pub request_id: Option<String>,
-  pub gateway_request_id: Option<String>,
-
-  pub error: Option<String>,
-
-  /// Payload of the webhook, if any.
-  pub payload: Option<Value>,
-}
-
-#[derive(Debug, Deserialize, ToSchema)]
-pub enum FalWebhookStatus {
-  #[serde(alias = "OK")]
-  Ok,
-  #[serde(alias = "ERROR")]
-  Error
-}
-
 // =============== Error Response ===============
 
 #[derive(Debug, Serialize, ToSchema)]
@@ -82,68 +61,60 @@ impl From<anyhow::Error> for FalWebhookError {
 
 // =============== Handler ===============
 
-// /// Fal webhook
-// #[utoipa::path(
-//   post,
-//   tag = "Webhooks",
-//   path = "/v1/webhooks/fal",
-//   responses(
-//     (status = 200, description = "Success", body = SimpleGenericJsonSuccess),
-//     (status = 400, description = "Bad input", body = FalWebhookError),
-//     (status = 401, description = "Not authorized", body = FalWebhookError),
-//     (status = 500, description = "Server error", body = FalWebhookError),
-//   ),
-//   params(
-//     ("request" = FalWebhookRequest, description = "Payload for Request"),
-//   )
-// )]
+// TODO(bt, 2025-06-03): Handle webhook crypto authentication
 pub async fn fal_webhook_handler(
   http_request: HttpRequest,
   request_body_bytes: Bytes,
   server_state: web::Data<Arc<ServerState>>,
-  //request: Json<FalWebhookRequest>,
 ) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
 
-  let webhook_payload = String::from_utf8(request_body_bytes.to_vec())
+  // Step 1: Parse bytes into a UTF-8 string and log it.
+  let raw_body = String::from_utf8(request_body_bytes.to_vec())
       .map_err(|err| {
-        error!("Fal Webhook: could not decode request body to UTF-8: {:?}", err);
+        error!("FAL webhook: could not decode request body to UTF-8: {:?}", err);
+        enqueue_parse_error_alert(&server_state, &http_request, "UTF-8 decode failed", &err, None);
         FalWebhookError::BadInput("Could not decode request body to UTF-8".to_string())
       })?;
 
-  info!("Received FAL webhook body: {}", webhook_payload);
+  info!("Received FAL webhook body: {}", raw_body);
 
-  let webhook_payload = WebhookPayload::try_from_json(&webhook_payload)
+  // Step 2: Parse into WebhookPayload.
+  let webhook_payload = parse_webhook_payload(&raw_body)
       .map_err(|err| {
-        error!("Fal Webhook: could not parse webhook payload as JSON: {:?}", err);
-        FalWebhookError::BadInput("Could not parse webhook payload as JSON".to_string())
+        error!("FAL webhook: could not parse webhook payload: {:?}", err);
+        enqueue_parse_error_alert(&server_state, &http_request, "JSON parse failed", &err, Some(&raw_body));
+        FalWebhookError::BadInput("Could not parse webhook payload".to_string())
       })?;
 
   let request_id = webhook_payload.request_id.as_str();
-  let status = webhook_payload.status;
 
-  info!("FAL webhook request_id: {} (status: {:?})", request_id, status);
+  info!("FAL webhook request_id: {} (status: {:?})", request_id, webhook_payload.status);
 
-  // TODO(bt): Longer term, we should just use `fal_client` to parse webhooks and add lots of integration tests
-  //  across dozens of real messages.
-  let payload = webhook_payload.payload
-      .as_ref()
-      .ok_or_else(|| {
-        warn!("FAL webhook missing payload for request_id: {}", request_id);
-        FalWebhookError::BadInput("Missing payload".to_string())
-      })?;
+  // Step 3: Parse the inner payload.
+  let inner_payload = parse_webhook_inner_payload(&webhook_payload);
 
-  let result = match status {
-    WebhookStatus::Ok => {
-      handle_sucessful_fal_webhook(&server_state, request_id, payload).await
+  // Step 4 & 5: Branch on the inner payload type.
+  let result = match inner_payload {
+    WebhookInnerPayload::Success(success_data) => {
+      handle_successful_fal_webhook(&server_state, request_id, &success_data.payload).await
     }
-    WebhookStatus::Error => {
+    WebhookInnerPayload::Error(error_data) => {
       handle_failed_fal_webhook(
         &server_state,
         request_id,
-        payload,
+        &error_data,
         webhook_payload.error.as_deref(),
       ).await
     }
+    WebhookInnerPayload::PayloadError(payload_error_data) => {
+      warn!(
+        "FAL webhook payload_error for request_id {}: {}",
+        request_id, payload_error_data.payload_error,
+      );
+      // Treat payload encoding errors as server errors — the request was OK but
+      // FAL couldn't encode the payload, so we can't process it.
+      Err(FalWebhookError::ServerError)
+    }
   };
 
   if let Err(ref err) = result {
@@ -152,8 +123,8 @@ pub async fn fal_webhook_handler(
     let notification = NotificationDetailsBuilder::from_error(err)
         .set_title("FAL webhook processing failed".to_string())
         .set_description(Some(format!(
-          "FAL webhook failed for request_id: {}\n\nError: {:?}",
-          request_id, err,
+          "FAL webhook failed for request_id: {}\n\nError: {:?}\n\nWebhook JSON Payload: {}",
+          request_id, err, raw_body,
         )))
         .set_third_party_id(Some(request_id.to_string()))
         .set_urgency(Some(NotificationUrgency::High))
@@ -168,3 +139,29 @@ pub async fn fal_webhook_handler(
 
   result
 }
+
+/// Send a pager alert for early parse failures (before we have a request_id).
+fn enqueue_parse_error_alert<E: std::fmt::Debug>(
+  server_state: &ServerState,
+  http_request: &HttpRequest,
+  context: &str,
+  err: &E,
+  maybe_raw_body: Option<&str>,
+) {
+  let description = match maybe_raw_body {
+    Some(body) => format!("Error: {:?}\n\nWebhook JSON Payload: {}", err, body),
+    None => format!("Error: {:?}", err),
+  };
+
+  let notification = NotificationDetailsBuilder::from_title(
+        format!("FAL webhook parse failure: {}", context))
+      .set_description(Some(description))
+      .set_urgency(Some(NotificationUrgency::High))
+      .set_http_method(Some(http_request.method().to_string()))
+      .set_http_path(Some(http_request.path().to_string()))
+      .build();
+
+  if let Err(pager_err) = server_state.pager.enqueue_page(notification) {
+    error!("Failed to enqueue FAL webhook parse error alert: {:?}", pager_err);
+  }
+}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/fal_error_detail.rs

@@ -1,142 +0,0 @@
-use serde_json::Value;
-
-/// A single error detail item from a FAL webhook error payload.
-///
-/// See: https://fal.ai/docs/documentation/model-apis/errors
-#[derive(Debug, Clone)]
-pub struct FalErrorDetail {
-  /// Human-readable error message. Should not be parsed for logic.
-  pub msg: Option<String>,
-
-  /// Machine-readable error type (e.g. "image_too_small", "image_too_large").
-  pub error_type: Option<String>,
-}
-
-/// Attempt to parse the "detail" array from a FAL webhook payload.
-///
-/// The payload may contain a "detail" key with an array of error objects,
-/// each having optional "msg" and "type" fields.
-///
-/// Returns an empty vec if the "detail" key is missing or not an array.
-pub fn parse_fal_error_details(payload: &Value) -> Vec<FalErrorDetail> {
-  let detail_array = match payload.get("detail").and_then(|v| v.as_array()) {
-    Some(arr) => arr,
-    None => return Vec::new(),
-  };
-
-  detail_array
-    .iter()
-    .map(|item| FalErrorDetail {
-      msg: item.get("msg").and_then(|v| v.as_str()).map(|s| s.to_string()),
-      error_type: item.get("type").and_then(|v| v.as_str()).map(|s| s.to_string()),
-    })
-    .collect()
-}
-
-/// Build a human-readable summary of the error details for logging and alerting.
-pub fn summarize_fal_error_details(details: &[FalErrorDetail]) -> String {
-  if details.is_empty() {
-    return "No error details provided.".to_string();
-  }
-
-  details
-    .iter()
-    .enumerate()
-    .map(|(i, detail)| {
-      let error_type = detail.error_type.as_deref().unwrap_or("unknown");
-      let msg = detail.msg.as_deref().unwrap_or("no message");
-      format!("[{}] type={}, msg={}", i + 1, error_type, msg)
-    })
-    .collect::<Vec<_>>()
-    .join("\n")
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use serde_json::json;
-
-  #[test]
-  fn parse_typical_error_detail() {
-    let payload = json!({
-      "detail": [
-        {
-          "loc": ["body", "image_url"],
-          "msg": "Image too small",
-          "type": "image_too_small",
-          "url": "https://docs.fal.ai/errors/#image_too_small",
-          "ctx": { "min_height": 512, "min_width": 512 }
-        }
-      ]
-    });
-
-    let details = parse_fal_error_details(&payload);
-    assert_eq!(details.len(), 1);
-    assert_eq!(details[0].msg.as_deref(), Some("Image too small"));
-    assert_eq!(details[0].error_type.as_deref(), Some("image_too_small"));
-  }
-
-  #[test]
-  fn parse_multiple_error_details() {
-    let payload = json!({
-      "detail": [
-        { "msg": "First error", "type": "err_one" },
-        { "msg": "Second error", "type": "err_two" }
-      ]
-    });
-
-    let details = parse_fal_error_details(&payload);
-    assert_eq!(details.len(), 2);
-    assert_eq!(details[0].error_type.as_deref(), Some("err_one"));
-    assert_eq!(details[1].error_type.as_deref(), Some("err_two"));
-  }
-
-  #[test]
-  fn parse_missing_detail_field() {
-    let payload = json!({ "images": [] });
-    let details = parse_fal_error_details(&payload);
-    assert!(details.is_empty());
-  }
-
-  #[test]
-  fn parse_detail_not_array() {
-    let payload = json!({ "detail": "some string" });
-    let details = parse_fal_error_details(&payload);
-    assert!(details.is_empty());
-  }
-
-  #[test]
-  fn parse_detail_with_missing_fields() {
-    let payload = json!({
-      "detail": [
-        { "msg": "Only message" },
-        { "type": "only_type" },
-        {}
-      ]
-    });
-
-    let details = parse_fal_error_details(&payload);
-    assert_eq!(details.len(), 3);
-    assert_eq!(details[0].msg.as_deref(), Some("Only message"));
-    assert_eq!(details[0].error_type, None);
-    assert_eq!(details[1].msg, None);
-    assert_eq!(details[1].error_type.as_deref(), Some("only_type"));
-    assert_eq!(details[2].msg, None);
-    assert_eq!(details[2].error_type, None);
-  }
-
-  #[test]
-  fn summarize_empty() {
-    assert_eq!(summarize_fal_error_details(&[]), "No error details provided.");
-  }
-
-  #[test]
-  fn summarize_single() {
-    let details = vec![FalErrorDetail {
-      msg: Some("Image too small".to_string()),
-      error_type: Some("image_too_small".to_string()),
-    }];
-    let summary = summarize_fal_error_details(&details);
-    assert_eq!(summary, "[1] type=image_too_small, msg=Image too small");
-  }
-}

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/handle_failed_fal_webhook.rs

@@ -3,55 +3,47 @@ use log::{error, info, warn};
 use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::get_inference_job_by_fal_id;
 use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs};
 use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
-use serde_json::Value;
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
+use fal_client::webhook_api::payload::webhook_error_type::WebhookErrorType;
+use fal_client::webhook_api::payload::webhook_inner_payload::ErrorData;
 use crate::http_server::endpoints::webhooks::fal_webhook_handler::FalWebhookError;
-use crate::http_server::endpoints::webhooks::process_failure::fal_error_detail::{FalErrorDetail, parse_fal_error_details, summarize_fal_error_details};
 use crate::state::server_state::ServerState;
 
-/// Map FAL error type strings to frontend failure categories.
+/// Map a FAL WebhookErrorType to a frontend failure category.
 ///
-/// Returns `GenerationFailed` as the default if no error details match a known type.
-fn guess_failure_category(details: &[FalErrorDetail]) -> FrontendFailureCategory {
-  for detail in details {
-    if let Some(error_type) = &detail.error_type {
-      match error_type.as_str() {
-        "content_policy_violation" => return FrontendFailureCategory::RuleBansUserContent,
-        "face_detection_error" => return FrontendFailureCategory::FaceNotDetected,
-        // These all map to GenerationFailed for now, but are listed explicitly
-        // so we can refine them later.
-        "no_media_generated"
-        | "image_too_small"
-        | "image_too_large"
-        | "image_load_error"
-        | "file_download_error"
-        | "file_too_large" => return FrontendFailureCategory::GenerationFailed,
-        _ => {}
-      }
-    }
+/// Returns `GenerationFailed` as the default if the error type is None or unrecognized.
+fn guess_failure_category(error_type: Option<&WebhookErrorType>) -> FrontendFailureCategory {
+  match error_type {
+    Some(WebhookErrorType::ContentPolicyViolation) => FrontendFailureCategory::RuleBansUserContent,
+    Some(WebhookErrorType::FaceDetectionError) => FrontendFailureCategory::FaceNotDetected,
+    // These all map to GenerationFailed for now, but are listed explicitly,
+    // so we can refine them later.
+    Some(WebhookErrorType::NoMediaGenerated)
+    | Some(WebhookErrorType::ImageTooSmall)
+    | Some(WebhookErrorType::ImageTooLarge)
+    | Some(WebhookErrorType::ImageLoadError)
+    | Some(WebhookErrorType::FileDownloadError)
+    | Some(WebhookErrorType::FileTooLarge) => FrontendFailureCategory::GenerationFailed,
+    _ => FrontendFailureCategory::GenerationFailed,
   }
-  FrontendFailureCategory::GenerationFailed
 }
 
 /// Handle a FAL webhook with status ERROR.
 ///
-/// Parses the error details from the payload, looks up the job, and marks it as failed.
+/// Looks up the job by request_id and marks it as failed using the parsed error data.
 pub async fn handle_failed_fal_webhook(
   server_state: &ServerState,
   request_id: &str,
-  payload: &Value,
+  error_data: &ErrorData,
   maybe_top_level_error: Option<&str>,
 ) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
 
-  // Parse the "detail" array from the payload.
-  let error_details = parse_fal_error_details(payload);
-  let error_summary = summarize_fal_error_details(&error_details);
-
   info!(
-    "FAL webhook ERROR for request_id {}: top_level_error={:?}, details=[{}]",
+    "FAL webhook ERROR for request_id {}: top_level_error={:?}, error_type={:?}, message={:?}",
     request_id,
     maybe_top_level_error,
-    error_summary,
+    error_data.error_type,
+    error_data.message,
   );
 
   // Look up the job record.
@@ -67,12 +59,9 @@ pub async fn handle_failed_fal_webhook(
     }
   };
 
-  // Build a failure reason from the error details or the top-level error string.
-  let public_failure_reason = if !error_details.is_empty() {
-    error_details.iter()
-        .filter_map(|d| d.msg.as_deref())
-        .collect::<Vec<_>>()
-        .join("; ")
+  // Build a failure reason from the error data or the top-level error string.
+  let public_failure_reason = if let Some(msg) = &error_data.message {
+    msg.clone()
   } else if let Some(top_level) = maybe_top_level_error {
     top_level.to_string()
   } else {
@@ -80,13 +69,14 @@ pub async fn handle_failed_fal_webhook(
   };
 
   let internal_failure_reason = format!(
-    "FAL request_id={}, top_level_error={:?}, details=[{}]",
+    "FAL request_id={}, top_level_error={:?}, error_type={:?}, message={:?}",
     request_id,
     maybe_top_level_error,
-    error_summary,
+    error_data.error_type,
+    error_data.message,
   );
 
-  let failure_category = guess_failure_category(&error_details);
+  let failure_category = guess_failure_category(error_data.error_type.as_ref());
 
   info!(
     "Marking job {} as failed for request_id {}. Category: {:?}, Reason: {}",

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_failure/mod.rs

@@ -1,2 +1 @@
-pub mod fal_error_detail;
 pub mod handle_failed_fal_webhook;

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/process_success/handle_successful_fal_webhook.rs

@@ -14,7 +14,7 @@ use super::process_model_glb_payload::process_model_glb_payload;
 use super::process_model_mesh_payload::process_model_mesh_payload;
 use super::process_video_payload::process_video_payload;
 
-pub async fn handle_sucessful_fal_webhook(
+pub async fn handle_successful_fal_webhook(
   server_state: &ServerState,
   request_id: &str,
   payload: &Value,