advanced http errors (#1287)

.claude/rules/code-style.md

@@ -20,10 +20,13 @@ then the story, then the fine print.
 ### Ordering within a file
 
 1. **Constants** at the top, right after imports.
-2. **Structs, enums, and type definitions** next. If a struct contains another struct, place
-   the outer/larger struct above the inner/smaller one.
-3. **Type ordering for API types**: Request, then Response (success), then Error.
-4. **`impl` blocks** after their type definitions.
+2. **The primary public type** (the main struct/enum the file is about) comes next.
+   A reader opening this file should immediately see what it defines.
+3. **Supporting/subordinate types** (sub-structs, helper enums, serialization DTOs) go
+   below the primary type, not above it. Ask: "Would a reader need to know this type
+   exists before seeing the main type?" — if no, push it down.
+4. **Type ordering for API types**: Request, then Response (success), then Error.
+5. **`impl` blocks** after their type definitions.
 
 ### Ordering within an `impl` block
 

crates/lib/pager/src/notification/notification_details.rs

@@ -1,7 +1,6 @@
 use chrono::{DateTime, Utc};
 
 use crate::notification::generate_deduplication_key::generate_deduplication_key;
-use crate::notification::notification_details_builder::NotificationDetailsBuilder;
 use crate::notification::notification_urgency::NotificationUrgency;
 
 /// Details for a pager notification.
@@ -20,7 +19,7 @@ pub struct NotificationDetails {
   pub(crate) event_time: DateTime<Utc>,
 
   /// The error that triggered this notification, if any.
-  pub(crate) maybe_error: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
+  pub(crate) maybe_error: Option<std::sync::Arc<dyn std::error::Error + Send + Sync + 'static>>,
 
   /// Whether this notification originated from an error.
   pub(crate) is_from_error: bool,

crates/lib/pager/src/notification/notification_details_builder.rs

@@ -1,5 +1,7 @@
-use chrono::{DateTime, Utc};
 use std::fmt::{Debug, Display};
+use std::sync::Arc;
+
+use chrono::{DateTime, Utc};
 
 use crate::notification::notification_details::NotificationDetails;
 use crate::notification::notification_urgency::NotificationUrgency;
@@ -10,7 +12,7 @@ pub struct NotificationDetailsBuilder {
   pub(crate) urgency: Option<NotificationUrgency>,
   pub(crate) event_time: DateTime<Utc>,
 
-  pub(crate) maybe_error: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
+  pub(crate) maybe_error: Option<Arc<dyn std::error::Error + Send + Sync + 'static>>,
   pub(crate) is_from_error: bool,
 
   pub(crate) http_method: Option<String>,
@@ -43,11 +45,13 @@ impl NotificationDetailsBuilder {
     }
   }
 
-  /// Create a builder from a boxed error.
-  ///
-  /// Sets the title to a placeholder and attaches the error object.
-  /// Callers should chain `.set_title()` to provide a meaningful title.
-  pub fn from_error(error: Box<dyn std::error::Error + Send + Sync + 'static>) -> Self {
+  /// Create a builder from a boxed error. Converts to `Arc` internally for shareability.
+  pub fn from_boxed_error(error: Box<dyn std::error::Error + Send + Sync + 'static>) -> Self {
+    Self::from_error(Arc::from(error))
+  }
+
+  /// Create a builder from an `Arc`'d error.
+  pub fn from_error(error: Arc<dyn std::error::Error + Send + Sync + 'static>) -> Self {
     Self {
       title: "Notification from Error".to_string(),
       description: None,
@@ -80,7 +84,7 @@ impl NotificationDetailsBuilder {
     self
   }
 
-  pub fn set_error(mut self, error: Option<Box<dyn std::error::Error + Send + Sync + 'static>>) -> Self {
+  pub fn set_error(mut self, error: Option<Arc<dyn std::error::Error + Send + Sync + 'static>>) -> Self {
     self.maybe_error = error;
     self
   }

crates/service/job/seedance2_pro_job/src/jobs/alert_on_error.rs

@@ -13,7 +13,7 @@ pub fn alert_pager_and_return_err<T>(
 ) -> anyhow::Result<T> {
   let err_message = format!("{:#}", err);
 
-  let mut builder = NotificationDetailsBuilder::from_error(err.into())
+  let mut builder = NotificationDetailsBuilder::from_boxed_error(err.into())
       .set_title(title.to_string())
       .set_urgency(Some(NotificationUrgency::Medium));
 

crates/service/job/seedance2_pro_job/src/jobs/process_job/process_failed_job.rs

@@ -85,7 +85,7 @@ pub async fn process_failed_job(
             err,
           );
 
-          let notification = NotificationDetailsBuilder::from_error(err.into())
+          let notification = NotificationDetailsBuilder::from_boxed_error(err.into())
               .set_title("Seedance2Pro refund failed".to_string())
               .set_inference_job_token(Some(job.job_token.to_string()))
               .set_third_party_id(Some(job.order_id.to_string()))
@@ -140,7 +140,7 @@ pub async fn process_failed_job(
       err
     );
 
-    let notification = NotificationDetailsBuilder::from_error(err.into())
+    let notification = NotificationDetailsBuilder::from_boxed_error(err.into())
         .set_title("Seedance2Pro mark job failed error".to_string())
         .set_urgency(Some(NotificationUrgency::Medium))
         .build();

crates/service/job/video_thumbnail_job/src/job/alert_on_error.rs

@@ -11,7 +11,7 @@ pub fn alert_pager_and_return_err<T>(
 ) -> anyhow::Result<T> {
   let err_message = format!("{:#}", err);
 
-  let notification = NotificationDetailsBuilder::from_error(err.into())
+  let notification = NotificationDetailsBuilder::from_boxed_error(err.into())
       .set_title(title.to_string())
       .set_urgency(Some(NotificationUrgency::Medium))
       .build();

crates/service/web/storyteller_web/src/http_server/common_responses/advanced_common_web_error.rs

@@ -0,0 +1,332 @@
+use std::fmt::{Display, Formatter};
+use std::sync::Arc;
+
+use actix_http::StatusCode;
+use actix_web::{HttpResponse, HttpResponseBuilder, ResponseError};
+
+use crate::http_server::web_utils::user_session::require_user_session::RequireUserSessionError;
+
+/// An error type for actix-web handlers that wraps causal errors for debugging
+/// and paging while presenting safe, generic HTTP responses to users.
+///
+/// ## Usage
+///
+/// ```ignore
+/// pub async fn my_handler(...) -> Result<Json<MyResponse>, AdvancedCommonWebError> {
+///   // sqlx errors, anyhow errors, session errors — all convert via ?
+///   let user = require_user_session_using_connection(&req, ...)?;
+///   let data = some_db_query(&pool).await?;
+///
+///   if data.is_none() {
+///     return Err(AdvancedCommonWebError::NotFound);
+///   }
+///
+///   Ok(Json(MyResponse { data }))
+/// }
+/// ```
+///
+/// Errors converted via `From<T>` become `UncaughtServerError` (always 500).
+/// The wrapped cause is never shown to users but is available to the error
+/// alerting middleware for paging and logging.
+pub enum AdvancedCommonWebError {
+  /// 400 Bad Request with a user-facing message.
+  BadInputWithSimpleMessage(String),
+  /// 401 Unauthorized.
+  NotAuthorized,
+  /// 404 Not Found.
+  NotFound,
+  /// 402 Payment Required.
+  PaymentRequired,
+
+  /// Uncaught errors are always 500 Internal Server Error.
+  /// The user will never see the error cause or message, but our
+  /// middleware will handle alerting, logging, etc.
+  ///
+  /// Stored in `Arc` so the error alerting middleware can clone it for paging
+  /// without consuming the original.
+  UncaughtServerError(Arc<dyn std::error::Error + Send + Sync + 'static>),
+}
+
+// =============== Public accessors ===============
+
+impl AdvancedCommonWebError {
+  /// Wrap any error as an `UncaughtServerError`.
+  pub fn from_error(error: impl std::error::Error + Send + Sync + 'static) -> Self {
+    Self::UncaughtServerError(Arc::new(error))
+  }
+
+  /// Extract the wrapped causal error (only present for `UncaughtServerError`).
+  pub fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
+    match self {
+      Self::UncaughtServerError(err) => Some(err.as_ref()),
+      _ => None,
+    }
+  }
+
+  /// Clone the wrapped causal error as an `Arc` (only present for `UncaughtServerError`).
+  /// Useful for passing the error to the pager system without consuming the original.
+  pub fn clone_cause_arc(&self) -> Option<Arc<dyn std::error::Error + Send + Sync + 'static>> {
+    match self {
+      Self::UncaughtServerError(err) => Some(Arc::clone(err)),
+      _ => None,
+    }
+  }
+
+  /// Whether this is a server error (500).
+  pub fn is_server_error(&self) -> bool {
+    matches!(self, Self::UncaughtServerError(_))
+  }
+}
+
+// =============== Display / Debug / Error ===============
+
+impl Display for AdvancedCommonWebError {
+  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+    match self {
+      Self::BadInputWithSimpleMessage(msg) => write!(f, "Bad input: {}", msg),
+      Self::NotAuthorized => write!(f, "Not authorized"),
+      Self::NotFound => write!(f, "Not found"),
+      Self::PaymentRequired => write!(f, "Payment required"),
+      Self::UncaughtServerError(err) => write!(f, "Server error: {}", err),
+    }
+  }
+}
+
+impl std::fmt::Debug for AdvancedCommonWebError {
+  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+    match self {
+      Self::BadInputWithSimpleMessage(msg) => write!(f, "BadInputWithSimpleMessage({:?})", msg),
+      Self::NotAuthorized => write!(f, "NotAuthorized"),
+      Self::NotFound => write!(f, "NotFound"),
+      Self::PaymentRequired => write!(f, "PaymentRequired"),
+      Self::UncaughtServerError(err) => write!(f, "UncaughtServerError({:?})", err),
+    }
+  }
+}
+
+impl std::error::Error for AdvancedCommonWebError {
+  fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+    match self {
+      Self::UncaughtServerError(err) => Some(err.as_ref()),
+      _ => None,
+    }
+  }
+}
+
+// =============== Actix ResponseError ===============
+
+impl ResponseError for AdvancedCommonWebError {
+  fn status_code(&self) -> StatusCode {
+    match self {
+      Self::BadInputWithSimpleMessage(_) => StatusCode::BAD_REQUEST,
+      Self::NotAuthorized => StatusCode::UNAUTHORIZED,
+      Self::NotFound => StatusCode::NOT_FOUND,
+      Self::PaymentRequired => StatusCode::PAYMENT_REQUIRED,
+      Self::UncaughtServerError(_) => StatusCode::INTERNAL_SERVER_ERROR,
+    }
+  }
+
+  fn error_response(&self) -> HttpResponse {
+    let status = self.status_code();
+
+    match self {
+      Self::BadInputWithSimpleMessage(msg) => {
+        HttpResponse::BadRequest()
+            .json(JsonErrorWithMessage {
+              success: false,
+              error_code: status.as_u16(),
+              error_code_str: status.canonical_reason(),
+              message: msg,
+            })
+      }
+      _ => {
+        HttpResponseBuilder::new(status)
+            .json(JsonErrorWithoutMessage {
+              success: false,
+              error_code: status.as_u16(),
+              error_code_str: status.canonical_reason(),
+            })
+      }
+    }
+  }
+}
+
+// =============== From impls (automatic ? conversion) ===============
+
+impl From<sqlx::Error> for AdvancedCommonWebError {
+  fn from(err: sqlx::Error) -> Self {
+    Self::from_error(err)
+  }
+}
+
+impl From<anyhow::Error> for AdvancedCommonWebError {
+  fn from(err: anyhow::Error) -> Self {
+    // anyhow::Error doesn't impl std::error::Error, so we go through Box -> Arc.
+    let boxed: Box<dyn std::error::Error + Send + Sync> = err.into();
+    Self::UncaughtServerError(Arc::from(boxed))
+  }
+}
+
+impl From<serde_json::Error> for AdvancedCommonWebError {
+  fn from(err: serde_json::Error) -> Self {
+    Self::from_error(err)
+  }
+}
+
+impl From<RequireUserSessionError> for AdvancedCommonWebError {
+  fn from(value: RequireUserSessionError) -> Self {
+    match value {
+      RequireUserSessionError::NotAuthorized => Self::NotAuthorized,
+      RequireUserSessionError::ServerError => Self::from_error(value),
+    }
+  }
+}
+
+// =============== Serialization helpers (private) ===============
+
+#[derive(Debug, Serialize)]
+struct JsonErrorWithoutMessage {
+  success: bool,
+  error_code: u16,
+  error_code_str: Option<&'static str>,
+}
+
+#[derive(Debug, Serialize)]
+struct JsonErrorWithMessage<'a> {
+  success: bool,
+  error_code: u16,
+  error_code_str: Option<&'static str>,
+  message: &'a str,
+}
+
+// =============== Tests ===============
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use actix_http::body::MessageBody;
+
+  #[test]
+  fn bad_input_returns_400_with_message() {
+    let error = AdvancedCommonWebError::BadInputWithSimpleMessage("name is required".to_string());
+    assert_eq!(error.status_code(), StatusCode::BAD_REQUEST);
+    assert!(!error.is_server_error());
+    assert!(error.cause().is_none());
+
+    let response = error.error_response();
+    let bytes = response.into_body().try_into_bytes().unwrap();
+    let body = String::from_utf8(bytes.to_vec()).unwrap();
+    assert!(body.contains("\"message\":\"name is required\""));
+    assert!(body.contains("\"error_code\":400"));
+  }
+
+  #[test]
+  fn not_found_returns_404() {
+    let error = AdvancedCommonWebError::NotFound;
+    assert_eq!(error.status_code(), StatusCode::NOT_FOUND);
+    assert!(!error.is_server_error());
+  }
+
+  #[test]
+  fn not_authorized_returns_401() {
+    let error = AdvancedCommonWebError::NotAuthorized;
+    assert_eq!(error.status_code(), StatusCode::UNAUTHORIZED);
+  }
+
+  #[test]
+  fn payment_required_returns_402() {
+    let error = AdvancedCommonWebError::PaymentRequired;
+    assert_eq!(error.status_code(), StatusCode::PAYMENT_REQUIRED);
+  }
+
+  #[test]
+  fn uncaught_io_error_returns_500_and_hides_cause() {
+    let io_err = std::io::Error::new(std::io::ErrorKind::Other, "disk exploded");
+    let error = AdvancedCommonWebError::from_error(io_err);
+
+    assert_eq!(error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
+    assert!(error.is_server_error());
+
+    // Cause is accessible for middleware
+    let cause = error.cause().unwrap();
+    assert_eq!(format!("{}", cause), "disk exploded");
+
+    // But never shown to the user in the HTTP response
+    let response = error.error_response();
+    let bytes = response.into_body().try_into_bytes().unwrap();
+    let body = String::from_utf8(bytes.to_vec()).unwrap();
+    assert!(!body.contains("disk exploded"));
+    assert!(body.contains("\"error_code\":500"));
+  }
+
+  #[test]
+  fn anyhow_error_converts_to_500_via_from() {
+    let error: AdvancedCommonWebError = anyhow::anyhow!("something broke in the pipeline").into();
+    assert_eq!(error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
+    assert!(error.is_server_error());
+
+    let cause = error.cause().unwrap();
+    assert!(format!("{}", cause).contains("something broke in the pipeline"));
+  }
+
+  #[test]
+  fn nested_anyhow_error_preserves_context_chain() {
+    let inner = std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "connection refused");
+    let with_context = anyhow::Error::new(inner).context("failed to connect to database");
+    let error: AdvancedCommonWebError = with_context.into();
+
+    assert!(error.is_server_error());
+    let cause = error.cause().unwrap();
+    let display = format!("{}", cause);
+    assert!(display.contains("failed to connect to database"));
+
+    // The original io error should be in the source chain
+    let source = cause.source();
+    assert!(source.is_some());
+  }
+
+  #[test]
+  fn require_user_session_not_authorized_maps_to_401() {
+    let error: AdvancedCommonWebError = RequireUserSessionError::NotAuthorized.into();
+    assert_eq!(error.status_code(), StatusCode::UNAUTHORIZED);
+    assert!(!error.is_server_error());
+  }
+
+  #[test]
+  fn require_user_session_server_error_wraps_cause() {
+    let error: AdvancedCommonWebError = RequireUserSessionError::ServerError.into();
+    assert_eq!(error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
+    assert!(error.is_server_error());
+    assert!(error.cause().is_some());
+    assert!(format!("{}", error.cause().unwrap()).contains("ServerError"));
+  }
+
+  #[test]
+  fn serde_json_error_converts_to_500() {
+    let bad_json = "not json at all{{{";
+    let serde_err: serde_json::Error = serde_json::from_str::<serde_json::Value>(bad_json).unwrap_err();
+    let error: AdvancedCommonWebError = serde_err.into();
+
+    assert_eq!(error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
+    assert!(error.is_server_error());
+    let cause = error.cause().unwrap();
+    assert!(format!("{}", cause).contains("expected"));
+  }
+
+  #[test]
+  fn error_source_returns_wrapped_error() {
+    let io_err = std::io::Error::new(std::io::ErrorKind::Other, "root cause");
+    let error = AdvancedCommonWebError::from_error(io_err);
+
+    let source = std::error::Error::source(&error);
+    assert!(source.is_some());
+    assert_eq!(format!("{}", source.unwrap()), "root cause");
+  }
+
+  #[test]
+  fn non_server_errors_have_no_source() {
+    assert!(std::error::Error::source(&AdvancedCommonWebError::NotFound).is_none());
+    assert!(std::error::Error::source(&AdvancedCommonWebError::NotAuthorized).is_none());
+    assert!(std::error::Error::source(&AdvancedCommonWebError::PaymentRequired).is_none());
+  }
+}

crates/service/web/storyteller_web/src/http_server/common_responses/mod.rs

@@ -1,3 +1,4 @@
+pub mod advanced_common_web_error;
 pub mod common_web_error;
 pub mod media;
 pub mod media_file_origin_details;

crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs

@@ -222,7 +222,7 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
           Err(err) => {
             warn!("Regular session fallback also failed for user {:?}: {:?}", user_token, err);
             if let Err(page_err) = server_state.pager.enqueue_page(
-              NotificationDetailsBuilder::from_error(err.into())
+              NotificationDetailsBuilder::from_boxed_error(err.into())
                   .set_title("Seedance 2.0 generation failed (whitelist + fallback)".to_string())
                   .set_urgency(Some(NotificationUrgency::High))
                   .build()
@@ -256,7 +256,7 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
       Err(err) => {
         warn!("Error calling seedance2pro generate_video: {:?}", err);
         if let Err(page_err) = server_state.pager.enqueue_page(
-          NotificationDetailsBuilder::from_error(err.into())
+          NotificationDetailsBuilder::from_boxed_error(err.into())
               .set_title("Seedance 2.0 generation failed".to_string())
               .set_urgency(Some(NotificationUrgency::High))
               .build()

crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs

@@ -129,7 +129,7 @@ pub async fn fal_webhook_handler(
         request_id, err, raw_body,
       );
 
-      let notification = NotificationDetailsBuilder::from_error(err.into())
+      let notification = NotificationDetailsBuilder::from_boxed_error(err.into())
           .set_title("FAL webhook processing failed".to_string())
           .set_description(Some(description))
           .set_third_party_id(Some(request_id.to_string()))

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_advanced_common_web_error.rs

@@ -0,0 +1,49 @@
+use log::{debug, warn};
+
+use pager::client::pager::Pager;
+use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
+
+use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
+
+/// Check `AdvancedCommonWebError` and alert on uncaught server errors.
+/// Returns `true` if the error was handled (alerted or intentionally skipped).
+pub(super) fn check_advanced_common_web_error(
+  pager: &Pager,
+  method: &str,
+  path: &str,
+  error: &AdvancedCommonWebError,
+) -> bool {
+  if !error.is_server_error() {
+    // Non-500 errors (400, 401, 404, 402) are intentional — don't alert.
+    return true;
+  }
+
+  let title = format!("UncaughtServerError on {} {}", method, path);
+
+  // Clone the Arc'd causal error so the notification owns a reference to it.
+  let cause_arc = error.clone_cause_arc();
+
+  let mut builder = if let Some(arc_err) = cause_arc {
+    NotificationDetailsBuilder::from_error(arc_err)
+        .set_title(title)
+  } else {
+    NotificationDetailsBuilder::from_title(title)
+  };
+
+  builder = builder
+      .set_http_method(Some(method.to_string()))
+      .set_http_path(Some(path.to_string()))
+      .set_http_status_code(Some(500))
+      .set_urgency(Some(NotificationUrgency::Medium));
+
+  let notification = builder.build();
+
+  if let Err(err) = pager.enqueue_page(notification) {
+    warn!("Error alerting middleware: failed to enqueue page: {:?}", err);
+  } else {
+    debug!("Error alerting middleware: enqueued alert for AdvancedCommonWebError::UncaughtServerError");
+  }
+
+  true
+}

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/error_alerting_middleware.rs

@@ -6,8 +6,10 @@ use actix_web::Error;
 
 use pager::client::pager::Pager;
 
+use crate::http_server::common_responses::advanced_common_web_error::AdvancedCommonWebError;
 use crate::http_server::common_responses::common_web_error::CommonWebError;
 use crate::state::flags::paging_flags::PagingFlags;
+use super::check_advanced_common_web_error::check_advanced_common_web_error;
 use super::check_common_web_error::check_common_web_error;
 use super::check_status_code_fallback::check_status_code_fallback;
 
@@ -121,6 +123,12 @@ fn check_ok_response_for_alerts<B>(
   if let Some(err) = response.response().error() {
     // --- Typed matchers (add new check_*.rs modules and downcast branches here) ---
 
+    if let Some(advanced_err) = err.as_error::<AdvancedCommonWebError>() {
+      if check_advanced_common_web_error(pager, method, path, advanced_err) {
+        return;
+      }
+    }
+
     if let Some(common_err) = err.as_error::<CommonWebError>() {
       if check_common_web_error(pager, method, path, common_err) {
         return;
@@ -141,6 +149,12 @@ fn check_err_for_alerts(
 ) {
   // --- Typed matchers (add new check_*.rs modules and downcast branches here) ---
 
+  if let Some(advanced_err) = err.as_error::<AdvancedCommonWebError>() {
+    if check_advanced_common_web_error(pager, method, path, advanced_err) {
+      return;
+    }
+  }
+
   if let Some(common_err) = err.as_error::<CommonWebError>() {
     if check_common_web_error(pager, method, path, common_err) {
       return;

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/mod.rs

@@ -1,3 +1,4 @@
+mod check_advanced_common_web_error;
 mod check_common_web_error;
 mod check_status_code_fallback;
 

crates/service/web/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs

@@ -35,7 +35,7 @@ pub async fn db_health_checker_thread(
           Ok(_) => {},
         }
 
-        let notification = NotificationDetailsBuilder::from_error(database_error.into())
+        let notification = NotificationDetailsBuilder::from_boxed_error(database_error.into())
             .set_title("DB health check thread failed check".to_string())
             .set_urgency(Some(NotificationUrgency::High))
             .build();