flags

crates/service/web/storyteller_web/config/storyteller-web.development.env

@@ -8,6 +8,10 @@ COOKIE_SECURE=false
 COOKIE_DOMAIN=.jungle.horse
 WEBSITE_HOMEPAGE_REDIRECT=http://jungle.horse
 
+# Paging
+ENABLE_PAGING=true
+ENABLE_PAGING_FOR_500S=true
+
 # Test the rate limiter bypass (obviously these are dummy values)
 AI_STREAMER_USERNAMES="PewDiePie,xQc"
 

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs

@@ -0,0 +1,36 @@
+use pager::client::pager::Pager;
+
+use crate::http_server::common_responses::common_web_error::CommonWebError;
+use super::enqueue_alert::enqueue_alert;
+
+/// Check `CommonWebError` and alert on server errors.
+/// Returns `true` if the error was handled (alerted or intentionally skipped).
+pub(super) fn check_common_web_error(
+  pager: &Pager,
+  method: &str,
+  path: &str,
+  error: &CommonWebError,
+) -> bool {
+  match error {
+    CommonWebError::ServerError => {
+      enqueue_alert(
+        pager,
+        format!("CommonWebError::ServerError on {} {}", method, path),
+        format!(
+          "A CommonWebError::ServerError was returned.\n\n\
+           Endpoint: {} {}\n\
+           Error: {:?}\n\
+           Time: {}",
+          method, path, error,
+          chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"),
+        ),
+      );
+      true
+    }
+    // Don't alert on client errors (400, 401, 404, 402).
+    CommonWebError::BadInputWithSimpleMessage(_) => true,
+    CommonWebError::NotFound => true,
+    CommonWebError::NotAuthorized => true,
+    CommonWebError::PaymentRequired => true,
+  }
+}

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs

@@ -0,0 +1,30 @@
+use pager::client::pager::Pager;
+
+use super::enqueue_alert::enqueue_alert;
+
+/// Fallback alerting based on HTTP status code when no typed error matched.
+pub(super) fn check_status_code_fallback(
+  pager: &Pager,
+  method: &str,
+  path: &str,
+  status_code: u16,
+) {
+  match status_code {
+    500 => {
+      enqueue_alert(
+        pager,
+        format!("HTTP 500: {} {}", method, path),
+        format!(
+          "An untyped HTTP 500 response was returned (no typed error matched).\n\n\
+           Endpoint: {} {}\n\
+           Time: {}",
+          method, path,
+          chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"),
+        ),
+      );
+    }
+    // Add more status-based rules here:
+    // 503 => { enqueue_alert(...); }
+    _ => {}
+  }
+}

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/enqueue_alert.rs

@@ -0,0 +1,14 @@
+use log::{debug, warn};
+
+use pager::client::pager::Pager;
+use pager::notification::notification_details::NotificationDetails;
+
+pub(super) fn enqueue_alert(pager: &Pager, summary: String, description: String) {
+  let notification = NotificationDetails::with_summary_and_description(summary, description);
+
+  if let Err(err) = pager.enqueue_page(notification) {
+    warn!("Error alerting middleware: failed to enqueue page: {:?}", err);
+  } else {
+    debug!("Error alerting middleware: enqueued alert");
+  }
+}

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/error_alerting_middleware.rs

@@ -0,0 +1,153 @@
+use std::future::{Future, Ready, ready};
+use std::pin::Pin;
+
+use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform};
+use actix_web::Error;
+
+use pager::client::pager::Pager;
+
+use crate::http_server::common_responses::common_web_error::CommonWebError;
+use crate::state::flags::paging_flags::PagingFlags;
+use super::check_common_web_error::check_common_web_error;
+use super::check_status_code_fallback::check_status_code_fallback;
+
+// ======================== Transform (factory) ========================
+
+/// Middleware that intercepts error responses and enqueues pager alerts.
+///
+/// Inspects errors in two ways:
+/// 1. **Typed error matching** via downcast (e.g. `CommonWebError::ServerError`)
+/// 2. **Status code fallback** for untyped 500s that slip through
+///
+/// Both `PagingFlags.is_paging_enabled` and `PagingFlags.is_paging_for_500s_enabled`
+/// must be true for any alerting to fire. Otherwise the middleware is a passthrough.
+///
+/// To add new error types, create a new `check_*.rs` matcher module and
+/// add a downcast branch in `check_ok_response_for_alerts` / `check_err_for_alerts`.
+#[derive(Clone)]
+pub struct ErrorAlertingMiddleware {
+  pager: Pager,
+  paging_flags: PagingFlags,
+}
+
+impl ErrorAlertingMiddleware {
+  pub fn new(pager: Pager, paging_flags: PagingFlags) -> Self {
+    Self { pager, paging_flags }
+  }
+}
+
+impl<S, B> Transform<S, ServiceRequest> for ErrorAlertingMiddleware
+  where
+      S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error> + 'static,
+      S::Future: 'static,
+      B: 'static,
+{
+  type Response = ServiceResponse<B>;
+  type Error = Error;
+  type InitError = ();
+  type Transform = ErrorAlertingService<S>;
+  type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+  fn new_transform(&self, service: S) -> Self::Future {
+    ready(Ok(ErrorAlertingService {
+      service,
+      pager: self.pager.clone(),
+      paging_flags: self.paging_flags.clone(),
+    }))
+  }
+}
+
+// ======================== Service (per-request) ========================
+
+pub struct ErrorAlertingService<S> {
+  service: S,
+  pager: Pager,
+  paging_flags: PagingFlags,
+}
+
+impl<S, B> Service<ServiceRequest> for ErrorAlertingService<S>
+  where
+      S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = Error> + 'static,
+      S::Future: 'static,
+      B: 'static,
+{
+  type Response = ServiceResponse<B>;
+  type Error = Error;
+  type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>>>>;
+
+  actix_service::forward_ready!(service);
+
+  fn call(&self, req: ServiceRequest) -> Self::Future {
+    // Fast path: if paging is disabled, skip all inspection.
+    if !self.paging_flags.is_paging_enabled || !self.paging_flags.is_paging_for_500s_enabled {
+      return Box::pin(self.service.call(req));
+    }
+
+    let method = req.method().to_string();
+    let path = req.path().to_string();
+    let pager = self.pager.clone();
+    let fut = self.service.call(req);
+
+    Box::pin(async move {
+      match fut.await {
+        Ok(res) => {
+          check_ok_response_for_alerts(&pager, &method, &path, &res);
+          Ok(res)
+        }
+        Err(err) => {
+          check_err_for_alerts(&pager, &method, &path, &err);
+          Err(err)
+        }
+      }
+    })
+  }
+}
+
+// ======================== Response inspection ========================
+
+/// Inspect a successful `ServiceResponse` that may contain an error response.
+///
+/// Actix-web converts `ResponseError` types into HTTP responses and stashes
+/// the original error in `response.error()`. We can downcast from there.
+fn check_ok_response_for_alerts<B>(
+  pager: &Pager,
+  method: &str,
+  path: &str,
+  response: &ServiceResponse<B>,
+) {
+  let status = response.status();
+
+  // Try to get the original typed error from the response (if it came from ResponseError).
+  if let Some(err) = response.response().error() {
+    // --- Typed matchers (add new check_*.rs modules and downcast branches here) ---
+
+    if let Some(common_err) = err.as_error::<CommonWebError>() {
+      if check_common_web_error(pager, method, path, common_err) {
+        return;
+      }
+    }
+  }
+
+  // --- Status code fallback ---
+  check_status_code_fallback(pager, method, path, status.as_u16());
+}
+
+/// Inspect an `actix_web::Error` returned from the handler or inner middleware.
+fn check_err_for_alerts(
+  pager: &Pager,
+  method: &str,
+  path: &str,
+  err: &Error,
+) {
+  // --- Typed matchers (add new check_*.rs modules and downcast branches here) ---
+
+  if let Some(common_err) = err.as_error::<CommonWebError>() {
+    if check_common_web_error(pager, method, path, common_err) {
+      return;
+    }
+  }
+
+  // --- Status code fallback ---
+  let status = err.as_response_error().status_code();
+  check_status_code_fallback(pager, method, path, status.as_u16());
+}

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/mod.rs

@@ -0,0 +1,5 @@
+mod check_common_web_error;
+mod check_status_code_fallback;
+mod enqueue_alert;
+
+pub mod error_alerting_middleware;

crates/service/web/storyteller_web/src/http_server/middleware/mod.rs

@@ -1 +1,2 @@
+pub mod error_alerting_middleware;
 pub mod pushback_filter_middleware;

crates/service/web/storyteller_web/src/main.rs

@@ -72,6 +72,7 @@ use crate::configs::app_startup::redis_rate_limiters::configure_redis_rate_limit
 use crate::configs::connect_to_database::connect_to_database;
 use crate::configs::static_api_tokens::StaticApiTokenSet;
 use crate::http_server::cookies::anonymous_visitor_tracking::avt_cookie_manager::AvtCookieManager;
+use crate::http_server::middleware::error_alerting_middleware::error_alerting_middleware::ErrorAlertingMiddleware;
 use crate::http_server::middleware::pushback_filter_middleware::PushbackFilter;
 use crate::http_server::routes::add_routes::add_routes;
 use crate::http_server::session::session_checker::SessionChecker;
@@ -338,7 +339,7 @@ async fn main() -> AnyhowResult<()> {
     ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
   };
 
-  let (pager, pager_worker) = build_pager(server_environment_typed);
+  let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed);
 
   info!("Spawning pager worker thread.");
 
@@ -391,6 +392,9 @@ async fn main() -> AnyhowResult<()> {
     // Disable voice features
     disable_tts: easyenv::get_env_bool_or_default("DISABLE_TTS", false),
     disable_voice_conversion: easyenv::get_env_bool_or_default("DISABLE_VOICE_CONVERSION", false),
+
+    // Paging
+    paging: paging_flags,
   };
 
   let third_party_url_redirector = ThirdPartyUrlRedirector::new(server_environment);
@@ -625,6 +629,15 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()>
     ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
   };
 
+  let paging_flags_for_middleware = server_state.flags.paging.clone();
+  let pager_for_middleware = server_state.pager.clone();
+  let enable_error_alerting = paging_flags_for_middleware.is_paging_enabled
+    && paging_flags_for_middleware.is_paging_for_500s_enabled;
+
+  if enable_error_alerting {
+    info!("Error alerting middleware is ENABLED (ENABLE_PAGING=true, ENABLE_PAGING_FOR_500S=true).");
+  }
+
   let server_state_arc = web::Data::new(Arc::new(server_state));
 
   let disabled_endpoints = read_disabled_endpoints();
@@ -682,6 +695,10 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()>
       .wrap(DefaultHeaders::new()
         .header("X-Backend-Hostname", &hostname)
         .header("X-Build-Sha", server_state_arc.server_info.build_sha.clone()))
+      .wrap(middleware::Condition::new(
+        enable_error_alerting,
+        ErrorAlertingMiddleware::new(pager_for_middleware.clone(), paging_flags_for_middleware.clone()),
+      ))
       .wrap(PushbackFilter::new(&server_state_arc.flags.clone()))
       .wrap(DisabledEndpointFilter::new(disabled_endpoints.clone()))
       .wrap(BannedIpFilter::new(ip_ban_list))

crates/service/web/storyteller_web/src/startup/build_pager.rs

@@ -4,20 +4,38 @@ use pager::client::pager_builder::PagerBuilder;
 use pager::worker::pager_worker::PagerWorker;
 use rootly_client::creds::rootly_api_key::RootlyApiKey;
 
-pub fn build_pager(server_environment: server_environment::ServerEnvironment) -> (Pager, PagerWorker) {
+use crate::state::flags::paging_flags::PagingFlags;
+
+pub fn build_pager(
+  server_environment: server_environment::ServerEnvironment,
+) -> (Pager, PagerWorker, PagingFlags) {
+  let paging_flags = PagingFlags::from_env();
+
+  info!("Paging flags: {:?}", paging_flags);
+
   let environment = if server_environment.is_deployed_in_production() {
     "production"
   } else {
     "development"
   };
 
+  // If paging is globally disabled, use a NoOp pager regardless of API key.
+  if !paging_flags.is_paging_enabled {
+    warn!("ENABLE_PAGING is false. Pager will be NoOp.");
+    let (pager, worker) = PagerBuilder::new()
+      .application_name("storyteller-web".to_string())
+      .environment(environment.to_string())
+      .build_with_worker();
+    return (pager, worker, paging_flags);
+  }
+
   let maybe_api_key = easyenv::get_env_string_optional("ROOTLY_API_KEY");
 
   let builder = PagerBuilder::new()
       .application_name("storyteller-web".to_string())
       .environment(environment.to_string());
 
-  match maybe_api_key {
+  let (pager, worker) = match maybe_api_key {
     Some(api_key) => {
       info!("Rootly API key found. Configuring pager with Rootly backend.");
       build_rootly_pager(builder, api_key)
@@ -26,10 +44,12 @@ pub fn build_pager(server_environment: server_environment::ServerEnvironment) ->
       warn!("ROOTLY_API_KEY not set. Pager will not send real pages.");
       builder.build_with_worker()
     }
-  }
+  };
+
+  (pager, worker, paging_flags)
 }
 
-fn build_rootly_pager(mut builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) {
+fn build_rootly_pager(builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) {
   let mut rootly_builder = builder
       .rootly(RootlyApiKey::new(api_key));
 

crates/service/web/storyteller_web/src/state/flags/mod.rs

@@ -0,0 +1 @@
+pub mod paging_flags;

crates/service/web/storyteller_web/src/state/flags/paging_flags.rs

@@ -0,0 +1,21 @@
+/// Flags controlling the pager system.
+#[derive(Clone, Debug)]
+pub struct PagingFlags {
+  /// Master switch for paging. If false, no pages are sent.
+  /// Env var: ENABLE_PAGING (default: false)
+  pub is_paging_enabled: bool,
+
+  /// If true, the error alerting middleware will enqueue pages for HTTP 500s.
+  /// Requires `is_paging_enabled` to also be true.
+  /// Env var: ENABLE_PAGING_FOR_500S (default: false)
+  pub is_paging_for_500s_enabled: bool,
+}
+
+impl PagingFlags {
+  pub fn from_env() -> Self {
+    Self {
+      is_paging_enabled: easyenv::get_env_bool_or_default("ENABLE_PAGING", false),
+      is_paging_for_500s_enabled: easyenv::get_env_bool_or_default("ENABLE_PAGING_FOR_500S", false),
+    }
+  }
+}

crates/service/web/storyteller_web/src/state/mod.rs

@@ -1,4 +1,5 @@
 pub mod cached_queries;
+pub mod flags;
 pub mod certs;
 pub mod memory_cache;
 pub mod server_state;

crates/service/web/storyteller_web/src/state/server_state.rs

@@ -33,6 +33,7 @@ use crate::http_server::endpoints::stats::result_transformer::CacheableQueueStat
 use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse;
 use crate::http_server::endpoints::voice_conversion::list_voice_conversion_models_handler::VoiceConversionModel;
 use crate::http_server::session::session_checker::SessionChecker;
+use crate::state::flags::paging_flags::PagingFlags;
 use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter;
 use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator;
 use crate::state::certs::google_sign_in_cert::GoogleSignInCert;
@@ -298,6 +299,9 @@ pub struct StaticFeatureFlags {
   /// Disable voice conversion endpoints
   /// If true, respond with 429.
   pub disable_voice_conversion: bool,
+
+  /// Paging system flags.
+  pub paging: PagingFlags,
 }
 
 /// Instead of top level service denial, these are bans against entities that instead return