health check thread pages

crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs

@@ -10,6 +10,17 @@ pub struct ModerationSendAlertRequest {
 
   /// Optional description for the alert. Defaults to "This is a test moderation alert."
   pub description: Option<String>,
+
+  /// Optional urgency of the page.
+  pub urgency: Option<ModerationSendAlertUrgency>,
+}
+
+#[derive(Deserialize, ToSchema, Copy, Clone, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum ModerationSendAlertUrgency {
+  High,
+  Medium,
+  Low,
 }
 
 #[derive(Serialize, ToSchema)]

crates/lib/pager/src/client/pager.rs

@@ -77,7 +77,7 @@ impl Pager {
     notification: NotificationDetails,
   ) -> Result<(), PagerError> {
     if self.client.is_noop() {
-      debug!("Pager no-op: would have enqueued page: {}", notification.summary);
+      debug!("Pager no-op: would have enqueued page: {}", notification.title);
       return Ok(());
     }
 
@@ -87,7 +87,7 @@ impl Pager {
     let dropped = queue.push(notification)?;
 
     if let Some(dropped) = dropped {
-      log::warn!("Pager queue overflow: dropped '{}'", dropped.summary);
+      log::warn!("Pager queue overflow: dropped '{}'", dropped.title);
     }
 
     Ok(())

crates/lib/pager/src/client/pager_client.rs

@@ -85,7 +85,7 @@ impl PagerClient {
   pub async fn send_page(&self, notification: &NotificationDetails) -> Result<Option<PageSentResult>, PagerError> {
     match &self.client_config {
       PagerClientConfig::NoOp => {
-        debug!("Pager no-op: would have sent page: {}", notification.summary);
+        debug!("Pager no-op: would have sent page: {}", notification.title);
         Ok(None)
       }
       PagerClientConfig::Rootly { .. } => {
@@ -113,7 +113,7 @@ impl PagerClient {
         .clone()
         .unwrap_or_else(|| "unknown".to_string());
 
-    debug!("Sending page via Rootly (source={}): {}", source, notification.summary);
+    debug!("Sending page via Rootly (source={}): {}", source, notification.title);
 
     let mut labels: Vec<(String, String)> = Vec::new();
 
@@ -190,7 +190,7 @@ impl PagerClient {
     let result = create_alert(CreateAlertArgs {
       api_key: api_key.clone(),
       source: "api".to_string(),
-      summary: notification.summary.clone(),
+      summary: notification.title.clone(),
       description,
       status: Some("triggered".to_string()),
       service_ids: self.service_id

crates/lib/pager/src/notification/generate_deduplication_key.rs

@@ -14,13 +14,13 @@ fn hours_since_epoch(time: DateTime<Utc>) -> i64 {
 /// The key is a hex-encoded SHA-256 hash of the notification's identifying fields.
 ///
 /// When both `http_method` and `endpoint_path` are present, they are included in the
-/// hash so that different endpoints with the same summary produce distinct keys.
+/// hash so that different endpoints with the same title produce distinct keys.
 /// The `hours_since_epoch` of the event time is included so that the same notification
 /// firing in different hours produces a different key (allowing re-alerting).
 pub(crate) fn generate_deduplication_key(details: &NotificationDetails) -> String {
   let mut hasher = Sha256::new();
 
-  hasher.update(details.summary.as_bytes());
+  hasher.update(details.title.as_bytes());
   hasher.update(if details.is_from_error { b"1" } else { b"0" });
 
   if let (Some(method), Some(path)) = (&details.http_method, &details.http_path) {
@@ -44,14 +44,14 @@ mod tests {
   use chrono::TimeZone;
 
   fn make_details(
-    summary: &str,
+    title: &str,
     is_from_error: bool,
     http_method: Option<&str>,
     http_path: Option<&str>,
     event_time: DateTime<Utc>,
   ) -> NotificationDetails {
     NotificationDetails {
-      summary: summary.to_string(),
+      title: title.to_string(),
       description: None,
       event_time,
       http_method: http_method.map(|s| s.to_string()),
@@ -154,7 +154,7 @@ mod tests {
     }
 
     #[test]
-    fn different_summary_different_key() {
+    fn different_title_different_key() {
       let time = Utc.with_ymd_and_hms(2026, 3, 30, 14, 0, 0).unwrap();
 
       let a = make_details("error A", true, None, None, time);
@@ -198,7 +198,7 @@ mod tests {
     use super::*;
 
     fn make_details_with_status(
-      summary: &str,
+      title: &str,
       is_from_error: bool,
       http_method: Option<&str>,
       http_path: Option<&str>,
@@ -206,7 +206,7 @@ mod tests {
       event_time: DateTime<Utc>,
     ) -> NotificationDetails {
       NotificationDetails {
-        summary: summary.to_string(),
+        title: title.to_string(),
         description: None,
         event_time,
         http_method: http_method.map(|s| s.to_string()),

crates/lib/pager/src/notification/notification_details.rs

@@ -8,7 +8,7 @@ use crate::notification::notification_urgency::NotificationUrgency;
 #[derive(Clone, Debug)]
 pub struct NotificationDetails {
   /// Title or summary of the alert.
-  pub summary: String,
+  pub title: String,
 
   /// Full details for the alert.
   pub description: Option<String>,
@@ -37,10 +37,10 @@ impl NotificationDetails {
     generate_deduplication_key(self)
   }
 
-  /// Create a notification with a summary and description.
-  pub fn with_summary_and_description(summary: String, description: String) -> Self {
+  /// Create a notification with a title and description.
+  pub fn with_title_and_description(title: String, description: String) -> Self {
     Self {
-      summary,
+      title,
       description: Some(description),
       event_time: Utc::now(),
       http_method: None,
@@ -51,10 +51,10 @@ impl NotificationDetails {
     }
   }
 
-  /// Create a notification with just a summary.
-  pub fn with_summary(summary: String) -> Self {
+  /// Create a notification with just a title.
+  pub fn with_title(title: String) -> Self {
     Self {
-      summary,
+      title,
       description: None,
       event_time: Utc::now(),
       http_method: None,
@@ -73,13 +73,13 @@ impl NotificationDetails {
   /// - A backtrace (if available via `std::backtrace`)
   /// - The timestamp of the event
   pub fn from_error<E: Debug + Display>(error: &E) -> Self {
-    let summary = format!("{}", error);
+    let title = format!("{}", error);
 
-    // Truncate summary to a reasonable length for alert titles.
-    let summary = if summary.len() > 200 {
-      format!("{}...", &summary[..197])
+    // Truncate the title to a reasonable length for alert titles.
+    let title = if title.len() > 200 {
+      format!("{}...", &title[..197])
     } else {
-      summary
+      title
     };
 
     let event_time = Utc::now();
@@ -115,7 +115,7 @@ impl NotificationDetails {
     let description = description_parts.join("\n");
 
     Self {
-      summary,
+      title,
       description: Some(description),
       event_time,
       http_method: None,
@@ -126,12 +126,12 @@ impl NotificationDetails {
     }
   }
 
-  /// Create a notification from an error, with a custom summary prefix.
+  /// Create a notification from an error, with a custom title prefix.
   pub fn from_error_with_context<E: Debug + Display>(context: &str, error: &E) -> Self {
     let mut notification = Self::from_error(error);
-    notification.summary = format!("{}: {}", context, notification.summary);
-    if notification.summary.len() > 200 {
-      notification.summary = format!("{}...", &notification.summary[..197]);
+    notification.title = format!("{}: {}", context, notification.title);
+    if notification.title.len() > 200 {
+      notification.title = format!("{}...", &notification.title[..197]);
     }
     notification
   }

crates/lib/pager/src/notification/notification_details_builder.rs

@@ -5,7 +5,7 @@ use crate::notification::notification_details::NotificationDetails;
 use crate::notification::notification_urgency::NotificationUrgency;
 
 pub struct NotificationDetailsBuilder {
-  summary: String,
+  title: String,
   description: Option<String>,
   event_time: DateTime<Utc>,
   http_method: Option<String>,
@@ -17,9 +17,9 @@ pub struct NotificationDetailsBuilder {
 
 impl NotificationDetailsBuilder {
   /// Create a builder from a summary string.
-  pub fn from_summary(summary: String) -> Self {
+  pub fn from_title(title: String) -> Self {
     Self {
-      summary,
+      title,
       description: None,
       event_time: Utc::now(),
       http_method: None,
@@ -38,7 +38,7 @@ impl NotificationDetailsBuilder {
     // TODO(bt,2026-03-30): Clean this up
     let details = NotificationDetails::from_error(error);
     Self {
-      summary: details.summary,
+      title: details.title,
       description: details.description,
       event_time: details.event_time,
       http_method: None,
@@ -49,8 +49,8 @@ impl NotificationDetailsBuilder {
     }
   }
   
-  pub fn set_summary(mut self, summary: String) -> Self {
-    self.summary = summary;
+  pub fn set_title(mut self, title: String) -> Self {
+    self.title = title;
     self
   }
 
@@ -81,7 +81,7 @@ impl NotificationDetailsBuilder {
 
   pub fn build(self) -> NotificationDetails {
     NotificationDetails {
-      summary: self.summary,
+      title: self.title,
       description: self.description,
       event_time: self.event_time,
       http_method: self.http_method,

crates/lib/pager/src/worker/pager_worker.rs

@@ -78,8 +78,8 @@ impl PagerWorker {
         match self.client.send_page(notification).await {
           Ok(Some(success)) => {
             debug!(
-              "Pager worker sent page: id={:?}, summary={}",
-              success.id, notification.summary
+              "Pager worker sent page: id={:?}, title={}",
+              success.id, notification.title
             );
           }
           Ok(None) => {
@@ -88,7 +88,7 @@ impl PagerWorker {
           Err(err) => {
             error!(
               "Pager worker failed to send page for '{}': {}",
-              notification.summary, err
+              notification.title, err
             );
             // Don't kill the thread on errors — keep processing.
           }

crates/lib/pager/src/worker/pager_worker_message_queue.rs

@@ -35,7 +35,7 @@ impl PagerWorkerMessageQueue {
   /// Push a notification onto the queue.
   ///
   /// If the queue is full, the oldest item is dropped and a warning is logged.
-  /// Returns `Ok(Some(summary))` if an old item was evicted, `Ok(None)` otherwise.
+  /// Returns `Ok(Some(notification))` if an old item was evicted, `Ok(None)` otherwise.
   pub fn push(&self, notification: NotificationDetails) -> Result<Option<NotificationDetails>, PagerError> {
     let mut queue = self.inner.lock()
       .map_err(|e| PagerSystemError::MutexPoisoned(format!("push: {}", e)))?;
@@ -43,7 +43,7 @@ impl PagerWorkerMessageQueue {
     let dropped = if queue.len() >= self.max_size {
       let old = queue.pop_front();
       if let Some(ref n) = old {
-        warn!("Pager queue full (max={}). Dropped oldest: {}", self.max_size, n.summary);
+        warn!("Pager queue full (max={}). Dropped oldest: {}", self.max_size, n.title);
       }
       old
     } else {

crates/service/web/storyteller_web/src/docs/api_doc.rs

@@ -691,6 +691,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
     EditUserFeatureFlagsRequest,
     ModerationSendAlertRequest,
     ModerationSendAlertResponse,
+    ModerationSendAlertUrgency,
     ModeratorListSubscribingUsersBySignupDateEntry,
     ModeratorListSubscribingUsersBySignupDateRequest,
     ModeratorListSubscribingUsersBySignupDateResponse,

crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs

@@ -223,7 +223,7 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
             warn!("Regular session fallback also failed for user {:?}: {:?}", user_token, err);
             if let Err(page_err) = server_state.pager.enqueue_page(
               NotificationDetailsBuilder::from_error(&err)
-                  .set_summary("Seedance 2.0 generation failed (whitelist + fallback)".to_string())
+                  .set_title("Seedance 2.0 generation failed (whitelist + fallback)".to_string())
                   .set_urgency(Some(NotificationUrgency::High))
                   .build()
             ) {
@@ -257,7 +257,7 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
         warn!("Error calling seedance2pro generate_video: {:?}", err);
         if let Err(page_err) = server_state.pager.enqueue_page(
           NotificationDetailsBuilder::from_error(&err)
-              .set_summary("Seedance 2.0 generation failed".to_string())
+              .set_title("Seedance 2.0 generation failed".to_string())
               .set_urgency(Some(NotificationUrgency::High))
               .build()
         ) {

crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs

@@ -4,13 +4,11 @@ use actix_web::web::Json;
 use actix_web::{web, HttpRequest};
 use log::warn;
 
-use artcraft_api_defs::moderation::alerts::moderation_send_alert::{
-  ModerationSendAlertRequest,
-  ModerationSendAlertResponse,
-};
+use artcraft_api_defs::moderation::alerts::moderation_send_alert::{ModerationSendAlertRequest, ModerationSendAlertResponse, ModerationSendAlertUrgency};
 use pager::client::pager::Pager;
 use pager::notification::notification_details::NotificationDetails;
-
+use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
 use crate::http_server::common_responses::common_web_error::CommonWebError;
 use crate::http_server::web_utils::user_session::require_moderator::{require_moderator, UseDatabase};
 use crate::state::server_state::ServerState;
@@ -33,17 +31,38 @@ pub async fn moderation_send_alert_handler(
   server_state: web::Data<Arc<ServerState>>,
 ) -> Result<Json<ModerationSendAlertResponse>, CommonWebError> {
 
-  let _user_session = require_moderator(&http_request, &server_state, UseDatabase::GrabNewConnection)
+  let user_session = require_moderator(&http_request, &server_state, UseDatabase::GrabNewConnection)
     .await
     .map_err(|_| CommonWebError::NotAuthorized)?;
 
   let title = request.title.clone()
-    .unwrap_or_else(|| "Test Moderation Alert".to_string());
+    .unwrap_or_else(|| "Moderation Alert".to_string());
 
-  let description = request.description.clone()
-    .unwrap_or_else(|| "This is a test moderation alert.".to_string());
+  let description = {
+    let description = request.description.clone()
+        .unwrap_or_else(|| "This moderation alert does not have a description.".to_string());
 
-  let notification = NotificationDetails::with_summary_and_description(title, description);
+    let description = vec![
+      description,
+      format!("Sent by: {}", user_session.username),
+    ];
+
+    description.join("\n\n")
+  };
+
+  let urgency = request.urgency
+      .map(|urgency| match urgency {
+        ModerationSendAlertUrgency::High => NotificationUrgency::High,
+        ModerationSendAlertUrgency::Medium => NotificationUrgency::Medium,
+        ModerationSendAlertUrgency::Low => NotificationUrgency::Low,
+      });
+
+  let notification = NotificationDetailsBuilder::from_title(title)
+      .set_description(Some(description))
+      .set_http_method(Some(http_request.method().to_string()))
+      .set_http_path(Some(http_request.path().to_string()))
+      .set_urgency(urgency)
+      .build();
 
   server_state.pager
     .enqueue_page(notification)

crates/service/web/storyteller_web/src/http_server/endpoints/service/health_check_handler.rs

@@ -10,6 +10,9 @@ use log::error;
 
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
 
+use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
+
 use crate::state::server_state::ServerState;
 
 // =============== Success Response ===============
@@ -73,6 +76,32 @@ pub async fn get_health_check_handler(
       .num_seconds()
       .unsigned_abs();
 
+  if !is_healthy {
+    let notification = NotificationDetailsBuilder::from_title(
+          format!("Health check unhealthy on {}", server_state.hostname))
+        .set_description(Some(format!(
+          "Health check returned unhealthy.\n\n\
+             Hostname: {}\n\
+             Build SHA: {}\n\
+             Seconds since startup: {}\n\
+             Unhealthy consecutive count: {:?}\n\
+             Last DB time: {:?}",
+          server_state.hostname,
+          server_state.server_info.build_sha,
+          seconds_since_startup,
+          health_check_status.unhealthy_check_consecutive_count,
+          health_check_status.last_db_time,
+        )))
+        .set_urgency(Some(NotificationUrgency::High))
+        .set_http_method(Some(http_request.method().to_string()))
+        .set_http_path(Some(http_request.path().to_string()))
+        .build();
+
+    if let Err(err) = server_state.pager.enqueue_page(notification) {
+      error!("Failed to enqueue health check alert: {:?}", err);
+    }
+  }
+
   let response = HealthCheckResponse {
     success: true,
     is_healthy: health_check_status.is_healthy,

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs

@@ -26,7 +26,7 @@ pub(super) fn check_common_web_error(
         chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"),
       );
 
-      let notification = NotificationDetailsBuilder::from_summary(summary)
+      let notification = NotificationDetailsBuilder::from_title(summary)
           .set_description(Some(description))
           .set_http_method(Some(method.to_string()))
           .set_http_path(Some(path.to_string()))

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs

@@ -22,7 +22,7 @@ pub(super) fn check_status_code_fallback(
         chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"),
       );
 
-      let notification = NotificationDetailsBuilder::from_summary(summary)
+      let notification = NotificationDetailsBuilder::from_title(summary)
           .set_description(Some(description))
           .set_http_method(Some(method.to_string()))
           .set_http_path(Some(path.to_string()))

crates/service/web/storyteller_web/src/main.rs

@@ -307,15 +307,34 @@ async fn main() -> AnyhowResult<()> {
   let mysql_pool4 = pool.clone();
   let mysql_pool5 = pool.clone();
 
+  let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?)
+      .ok_or(anyhow!("invalid server environment"))?;
+
+  let server_environment_typed = match server_environment {
+    ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
+    ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
+  };
+
+  let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed, &server_hostname);
+
   let tokio_runtime = Runtime::new()?;
 
+  info!("Spawning pager worker thread.");
+
+  tokio_runtime.spawn(async move {
+    pager_worker.run().await;
+  });
+
   info!("Spawning DB health checker thread.");
 
+  let health_checker_pager = pager.clone();
+
   tokio_runtime.spawn(async move {
     db_health_checker_thread(
       health_check_status2,
       mysql_pool3,
       health_check_interval,
+      health_checker_pager,
     ).await;
   });
 
@@ -331,22 +350,6 @@ async fn main() -> AnyhowResult<()> {
     poll_model_token_info_thread(model_token_info_cache2, mysql_pool5).await;
   });
 
-  let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?)
-      .ok_or(anyhow!("invalid server environment"))?;
-
-  let server_environment_typed = match server_environment {
-    ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
-    ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
-  };
-
-  let (pager, pager_worker, paging_flags) = build_pager(server_environment_typed, &server_hostname);
-
-  info!("Spawning pager worker thread.");
-
-  tokio_runtime.spawn(async move {
-    pager_worker.run().await;
-  });
-
   let stripe_configs = StripeConfig {
     checkout: StripeCheckoutConfigs {
       success_url: FullUrlOrPath::Path(easyenv::get_env_string_required("STRIPE_CHECKOUT_SUCCESS_URL_PATH")?),

crates/service/web/storyteller_web/src/threads/db_health_checker_thread/db_health_checker_thread.rs

@@ -4,6 +4,9 @@ use log::debug;
 use log::error;
 use log::warn;
 use mysql_queries::queries::health_check::health_check_query::health_check_db;
+use pager::client::pager::Pager;
+use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
 use sqlx::MySqlPool;
 
 use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus;
@@ -12,6 +15,7 @@ pub async fn db_health_checker_thread(
   health_check_status: HealthCheckStatus,
   mysql_pool: MySqlPool,
   check_duration: Duration,
+  pager: Pager,
 ) {
   loop {
     debug!("Checking DB health...");
@@ -23,12 +27,22 @@ pub async fn db_health_checker_thread(
           Ok(_) => {},
         }
       }
-      Err(e) => {
-        error!("Problem health checking database: {:?}", e);
+      Err(database_error) => {
+        error!("Problem health checking database: {:?}", database_error);
+
         match health_check_status.record_ping_failure() {
           Err(e) => error!("Problem updating application health checks!"),
           Ok(_) => {},
         }
+
+        let notification = NotificationDetailsBuilder::from_error(&database_error)
+            .set_title("DB health check thread failed check".to_string())
+            .set_urgency(Some(NotificationUrgency::High))
+            .build();
+
+        if let Err(page_err) = pager.enqueue_page(notification) {
+          error!("Failed to enqueue DB health check alert: {:?}", page_err);
+        }
       }
     }