more context for errors

crates/lib/pager/src/client/pager_client.rs

@@ -177,6 +177,22 @@ impl PagerClient {
           parts.push(format!("HTTP Status Code: {}", status_code));
         }
 
+        if let Some(user_token) = &notification.user_token {
+          parts.push(format!("User Token: {}", user_token));
+        }
+
+        if let Some(media_file_token) = &notification.media_file_token {
+          parts.push(format!("Media File Token: {}", media_file_token));
+        }
+
+        if let Some(inference_job_token) = &notification.inference_job_token {
+          parts.push(format!("Inference Job Token: {}", inference_job_token));
+        }
+
+        if let Some(third_party_id) = &notification.third_party_id {
+          parts.push(format!("Third Party ID: {}", third_party_id));
+        }
+
         if let Some(h) = &self.hostname {
           parts.push(format!("Hostname: {}", h));
         }

crates/lib/pager/src/notification/generate_deduplication_key.rs

@@ -59,6 +59,10 @@ mod tests {
       http_status_code: None,
       is_from_error,
       urgency: None,
+      user_token: None,
+      media_file_token: None,
+      inference_job_token: None,
+      third_party_id: None,
     }
   }
 
@@ -214,6 +218,10 @@ mod tests {
         http_status_code,
         is_from_error,
         urgency: None,
+        user_token: None,
+        media_file_token: None,
+        inference_job_token: None,
+        third_party_id: None,
       }
     }
 

crates/lib/pager/src/notification/notification_details.rs

@@ -30,6 +30,18 @@ pub struct NotificationDetails {
 
   /// Urgency level for the notification.
   pub urgency: Option<NotificationUrgency>,
+
+  /// User token associated with the event, if any.
+  pub user_token: Option<String>,
+
+  /// Media file token associated with the event, if any.
+  pub media_file_token: Option<String>,
+
+  /// Inference job token associated with the event, if any.
+  pub inference_job_token: Option<String>,
+
+  /// Third-party identifier associated with the event, if any.
+  pub third_party_id: Option<String>,
 }
 
 impl NotificationDetails {
@@ -48,6 +60,10 @@ impl NotificationDetails {
       http_status_code: None,
       is_from_error: false,
       urgency: None,
+      user_token: None,
+      media_file_token: None,
+      inference_job_token: None,
+      third_party_id: None,
     }
   }
 
@@ -62,6 +78,10 @@ impl NotificationDetails {
       http_status_code: None,
       is_from_error: false,
       urgency: None,
+      user_token: None,
+      media_file_token: None,
+      inference_job_token: None,
+      third_party_id: None,
     }
   }
 
@@ -123,6 +143,10 @@ impl NotificationDetails {
       http_status_code: None,
       is_from_error: true,
       urgency: None,
+      user_token: None,
+      media_file_token: None,
+      inference_job_token: None,
+      third_party_id: None,
     }
   }
 

crates/lib/pager/src/notification/notification_details_builder.rs

@@ -13,6 +13,10 @@ pub struct NotificationDetailsBuilder {
   http_status_code: Option<u16>,
   is_from_error: bool,
   urgency: Option<NotificationUrgency>,
+  user_token: Option<String>,
+  media_file_token: Option<String>,
+  inference_job_token: Option<String>,
+  third_party_id: Option<String>,
 }
 
 impl NotificationDetailsBuilder {
@@ -27,6 +31,10 @@ impl NotificationDetailsBuilder {
       http_status_code: None,
       is_from_error: false,
       urgency: None,
+      user_token: None,
+      media_file_token: None,
+      inference_job_token: None,
+      third_party_id: None,
     }
   }
 
@@ -46,6 +54,10 @@ impl NotificationDetailsBuilder {
       http_status_code: None,
       is_from_error: true,
       urgency: None,
+      user_token: None,
+      media_file_token: None,
+      inference_job_token: None,
+      third_party_id: None,
     }
   }
   
@@ -79,6 +91,26 @@ impl NotificationDetailsBuilder {
     self
   }
 
+  pub fn set_user_token(mut self, user_token: Option<String>) -> Self {
+    self.user_token = user_token;
+    self
+  }
+
+  pub fn set_media_file_token(mut self, media_file_token: Option<String>) -> Self {
+    self.media_file_token = media_file_token;
+    self
+  }
+
+  pub fn set_inference_job_token(mut self, inference_job_token: Option<String>) -> Self {
+    self.inference_job_token = inference_job_token;
+    self
+  }
+
+  pub fn set_third_party_id(mut self, third_party_id: Option<String>) -> Self {
+    self.third_party_id = third_party_id;
+    self
+  }
+
   pub fn build(self) -> NotificationDetails {
     NotificationDetails {
       title: self.title,
@@ -89,6 +121,10 @@ impl NotificationDetailsBuilder {
       http_status_code: self.http_status_code,
       is_from_error: self.is_from_error,
       urgency: self.urgency,
+      user_token: self.user_token,
+      media_file_token: self.media_file_token,
+      inference_job_token: self.inference_job_token,
+      third_party_id: self.third_party_id,
     }
   }
 }

crates/service/job/seedance2_pro_job/src/jobs/alert_on_error.rs

@@ -1,4 +1,5 @@
 use log::error;
+use mysql_queries::queries::generic_inference::seedance2pro::list_pending_seedance2pro_jobs::PendingSeedance2ProJob;
 use pager::client::pager::Pager;
 use pager::notification::notification_details_builder::NotificationDetailsBuilder;
 use pager::notification::notification_urgency::NotificationUrgency;
@@ -8,11 +9,20 @@ pub fn alert_pager_and_return_err<T>(
   pager: &Pager,
   title: &str,
   err: anyhow::Error,
+  job: Option<&PendingSeedance2ProJob>,
 ) -> anyhow::Result<T> {
-  let notification = NotificationDetailsBuilder::from_error(&err)
+  let mut builder = NotificationDetailsBuilder::from_error(&err)
       .set_title(title.to_string())
-      .set_urgency(Some(NotificationUrgency::Medium))
-      .build();
+      .set_urgency(Some(NotificationUrgency::Medium));
+
+  if let Some(job) = job {
+    builder = builder
+        .set_inference_job_token(Some(job.job_token.to_string()))
+        .set_third_party_id(Some(job.order_id.clone()))
+        .set_user_token(job.maybe_creator_user_token.as_ref().map(|t| t.to_string()));
+  }
+
+  let notification = builder.build();
 
   if let Err(pager_err) = pager.enqueue_page(notification) {
     error!("Failed to enqueue pager alert: {:?}", pager_err);

crates/service/job/seedance2_pro_job/src/jobs/main_loop.rs

@@ -21,7 +21,7 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
 
     if let Err(err) = result {
       error!("Error in poll iteration: {:?}", err);
-      let _ = alert_pager_and_return_err::<()>(&job_dependencies.pager, "Seedance2Pro poll iteration error", err);
+      let _ = alert_pager_and_return_err::<()>(&job_dependencies.pager, "Seedance2Pro poll iteration error", err, None);
       let _ = job_dependencies.job_stats.increment_failure_count();
     }
 
@@ -54,7 +54,7 @@ async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> {
     Ok(jobs) => jobs,
     Err(err) => {
       error!("Failed to list pending seedance2pro jobs: {:?}", err);
-      return alert_pager_and_return_err(&deps.pager, "Seedance2Pro DB query failed", err.into());
+      return alert_pager_and_return_err(&deps.pager, "Seedance2Pro DB query failed", err.into(), None);
     }
   };
 
@@ -108,6 +108,7 @@ async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> {
           &deps.pager,
           "Seedance2Pro API poll failed",
           anyhow::anyhow!("poll_orders failed: {:?}", err),
+          None,
         );
       }
     };

crates/service/job/seedance2_pro_job/src/jobs/process_job/process_failed_job.rs

@@ -87,6 +87,9 @@ pub async fn process_failed_job(
 
           let notification = NotificationDetailsBuilder::from_error(&err)
               .set_title("Seedance2Pro refund failed".to_string())
+              .set_inference_job_token(Some(job.job_token.to_string()))
+              .set_third_party_id(Some(job.order_id.to_string()))
+              .set_user_token(job.maybe_creator_user_token.as_ref().map(|t| t.to_string()))
               .set_urgency(Some(NotificationUrgency::Medium))
               .build();
 

crates/service/job/seedance2_pro_job/src/jobs/process_job/process_successful_job.rs

@@ -58,6 +58,7 @@ pub async fn process_successful_job(
           &deps.pager,
           "Seedance2Pro video download failed",
           anyhow!("error reading video bytes: {:?}", err),
+          Some(job),
         );
       }
     },
@@ -67,6 +68,7 @@ pub async fn process_successful_job(
         &deps.pager,
         "Seedance2Pro video download failed",
         anyhow!("reqwest error downloading video: {:?}", err),
+        Some(job),
       );
     }
   };
@@ -102,6 +104,7 @@ pub async fn process_successful_job(
       &deps.pager,
       "Seedance2Pro bucket upload failed",
       anyhow!("error uploading video to bucket: {:?}", err),
+      Some(job),
     );
   }
 
@@ -141,6 +144,7 @@ pub async fn process_successful_job(
         &deps.pager,
         "Seedance2Pro media file insert failed",
         anyhow!("error inserting media file record: {:?}", err),
+        Some(job),
       );
     }
   };
@@ -166,6 +170,7 @@ pub async fn process_successful_job(
       &deps.pager,
       "Seedance2Pro job completion update failed",
       anyhow!("error marking job done: {:?}", err),
+      Some(job),
     );
   }
 

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs

@@ -14,15 +14,17 @@ use actix_web::{web, HttpRequest, HttpResponse};
 use anyhow::Error;
 use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess;
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
-use log::{info, warn};
+use log::{error, info, warn};
 use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::get_inference_job_by_fal_id;
 use mysql_queries::queries::generic_inference::fal::mark_fal_generic_inference_job_successfully_done::{mark_fal_generic_inference_job_successfully_done, MarkJobArgs};
+use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
 use serde_json::Value;
 use utoipa::ToSchema;
 
 // 1. tauri --> hit endpoint to enqueue
 //
-// 2. webhook 
+// 2. webhook
 //  - upload media file
 //  - update job record with media token + status
 //
@@ -35,12 +37,12 @@ use utoipa::ToSchema;
 #[derive(Debug, Deserialize, ToSchema)]
 pub struct FalWebhookRequest {
   pub status: FalWebhookStatus,
-  
+
   pub request_id: Option<String>,
   pub gateway_request_id: Option<String>,
 
   pub error: Option<String>,
-  
+
   /// Payload of the webhook, if any.
   pub payload: Option<Value>,
 }
@@ -116,62 +118,97 @@ pub async fn fal_webhook_handler(
 ) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
 
   info!("Received FAL webhook body: {:?}", request);
-  
+
   let request_id = request.request_id
       .as_deref()
       .ok_or_else(|| FalWebhookError::BadInput("Missing request_id".to_string()))?;
-  
+
+  info!("FAL webhook request_id: {}", request_id);
+
   let payload = request.payload
       .as_ref()
-      .ok_or_else(|| FalWebhookError::BadInput("Missing payload".to_string()))?;
-  
+      .ok_or_else(|| {
+        warn!("FAL webhook missing payload for request_id: {}", request_id);
+        FalWebhookError::BadInput("Missing payload".to_string())
+      })?;
+
+  let result = process_fal_webhook(&server_state, request_id, payload).await;
+
+  if let Err(ref err) = result {
+    error!("FAL webhook error for request_id {}: {:?}", request_id, err);
+
+    let notification = NotificationDetailsBuilder::from_error(err)
+        .set_title("FAL webhook processing failed".to_string())
+        .set_description(Some(format!(
+          "FAL webhook failed for request_id: {}\n\nError: {:?}",
+          request_id, err,
+        )))
+        .set_third_party_id(Some(request_id.to_string()))
+        .set_urgency(Some(NotificationUrgency::High))
+        .set_http_method(Some(http_request.method().to_string()))
+        .set_http_path(Some(http_request.path().to_string()))
+        .build();
+
+    if let Err(pager_err) = server_state.pager.enqueue_page(notification) {
+      error!("Failed to enqueue FAL webhook pager alert: {:?}", pager_err);
+    }
+  }
+
+  result
+}
+
+async fn process_fal_webhook(
+  server_state: &ServerState,
+  request_id: &str,
+  payload: &Value,
+) -> Result<Json<SimpleGenericJsonSuccess>, FalWebhookError> {
+
   let db_result = get_inference_job_by_fal_id(
     request_id,
     &server_state.mysql_pool,
   ).await;
-  
+
   let job = match db_result {
     Ok(Some(record)) => record,
     Ok(None) => {
-      warn!("Could not find job record by fal request id: {:?} ; request payload = {:?}", request_id, request);
+      warn!("Could not find job record by fal request_id: {}", request_id);
       return Err(FalWebhookError::NotFound)
     },
     Err(err) => {
-      warn!("Error querying job record: {:?}", err);
+      warn!("Error querying job record for request_id {}: {:?}", request_id, err);
       return Err(FalWebhookError::ServerError);
     }
   };
 
-  info!("Fal webhook job record: {:?}", job);
+  info!("Fal webhook job record for request_id {}: {:?}", request_id, job);
 
   let mut maybe_media_token = None;
   let mut maybe_batch_token = None;
 
   if let Some(payload_obj) = payload.as_object() {
     if payload_obj.contains_key("image") {
-      info!("Handling image payload for job: {:?}", job.job_token);
-      let token = handle_image_payload(payload_obj, &job, &server_state).await?;
+      info!("Handling image payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = handle_image_payload(payload_obj, &job, server_state).await?;
       maybe_media_token = Some(token);
     } else if payload_obj.contains_key("images") {
-      (maybe_media_token, maybe_batch_token) = handle_images_payload(payload_obj, &job, &server_state).await?;
+      (maybe_media_token, maybe_batch_token) = handle_images_payload(payload_obj, &job, server_state).await?;
     } else if payload_obj.contains_key("video") {
-      info!("Handling video payload for job: {:?}", job.job_token);
-      let token = handle_video_payload(payload_obj, &job, &server_state).await?;
+      info!("Handling video payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = handle_video_payload(payload_obj, &job, server_state).await?;
       maybe_media_token = Some(token);
     } else if payload_obj.contains_key("model_glb") {
-      info!("Handling model_glb payload for job: {:?}", job.job_token);
-      let token = handle_model_glb_payload(payload_obj, &job, &server_state).await?;
+      info!("Handling model_glb payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = handle_model_glb_payload(payload_obj, &job, server_state).await?;
       maybe_media_token = Some(token);
     } else if payload_obj.contains_key("model_mesh") {
-      info!("Handling model_mesh payload for job: {:?}", job.job_token);
-      let token = handle_model_mesh_payload(payload_obj, &job, &server_state).await?;
+      info!("Handling model_mesh payload for request_id {} / job {:?}", request_id, job.job_token);
+      let token = handle_model_mesh_payload(payload_obj, &job, server_state).await?;
       maybe_media_token = Some(token);
     }
   }
-  
+
   if let Some(media_token) = maybe_media_token {
-    info!("Media file token: {:?}", media_token);
-    // TODO: Update job metadata.
+    info!("Media file token for request_id {}: {:?}", request_id, media_token);
     mark_fal_generic_inference_job_successfully_done(MarkJobArgs {
       job_token: &job.job_token,
       media_file_token: &media_token,
@@ -179,11 +216,11 @@ pub async fn fal_webhook_handler(
       mysql_executor: &server_state.mysql_pool,
       phantom: Default::default(),
     }).await.map_err(|err| {
-      warn!("Error marking job as successfully done: {:?}", err);
+      warn!("Error marking job as successfully done for request_id {}: {:?}", request_id, err);
       FalWebhookError::ServerError
     })?;
   } else {
-    warn!("No media token found in payload for job: {:?}", job.job_token);
+    warn!("No media token found in payload for request_id {} / job {:?}", request_id, job.job_token);
   }
 
   Ok(SimpleGenericJsonSuccess::wrapped(true))