more pager updates to fix spammy pager (#1260)

crates/api_clients/rootly_client/src/requests/create_alert/create_alert.rs

@@ -76,6 +76,7 @@ pub struct CreateAlertSuccess {
 
 // ======================== Implementation ========================
 
+/// https://docs.rootly.com/api-reference/alerts/creates-an-alert
 pub async fn create_alert(args: CreateAlertArgs) -> Result<CreateAlertSuccess, RootlyError> {
   let url = format!("{}/alerts", ROOTLY_API_BASE_URL);
 

crates/lib/pager/src/client/pager_builder.rs

@@ -14,6 +14,7 @@ pub struct PagerBuilder {
   application_name: Option<String>,
   environment: Option<String>,
   hostname: Option<String>,
+  service_id: Option<String>,
   queue_capacity: Option<usize>,
 }
 
@@ -24,6 +25,7 @@ impl PagerBuilder {
       application_name: None,
       environment: None,
       hostname: None,
+      service_id: None,
       queue_capacity: None,
     }
   }
@@ -46,12 +48,20 @@ impl PagerBuilder {
     self
   }
 
+  /// Set the service ID to associate alerts with.
+  pub fn service_id(mut self, service_id: String) -> Self {
+    self.service_id = Some(service_id);
+    self
+  }
+
   /// Configure the Rootly backend. Returns a sub-builder for Rootly-specific options.
   pub fn rootly(self, api_key: RootlyApiKey) -> RootlyConfigBuilder {
     RootlyConfigBuilder {
       parent: self,
       api_key,
-      alert_urgency_id: None,
+      urgency_id_high: None,
+      urgency_id_medium: None,
+      urgency_id_low: None,
       notification_target_type: None,
       notification_target_id: None,
     }
@@ -113,7 +123,7 @@ impl PagerBuilder {
       }
     };
 
-    PagerClient::new(client_config, self.application_name.clone(), self.environment.clone(), self.hostname.clone())
+    PagerClient::new(client_config, self.application_name.clone(), self.environment.clone(), self.hostname.clone(), self.service_id.clone())
   }
 }
 
@@ -122,15 +132,29 @@ impl PagerBuilder {
 pub struct RootlyConfigBuilder {
   parent: PagerBuilder,
   api_key: RootlyApiKey,
-  alert_urgency_id: Option<String>,
+  urgency_id_high: Option<String>,
+  urgency_id_medium: Option<String>,
+  urgency_id_low: Option<String>,
   notification_target_type: Option<String>,
   notification_target_id: Option<String>,
 }
 
 impl RootlyConfigBuilder {
-  /// Set the alert urgency ID.
-  pub fn alert_urgency_id(mut self, id: String) -> Self {
-    self.alert_urgency_id = Some(id);
+  /// Set the urgency ID for high-urgency alerts.
+  pub fn urgency_id_high(mut self, id: String) -> Self {
+    self.urgency_id_high = Some(id);
+    self
+  }
+
+  /// Set the urgency ID for medium-urgency alerts.
+  pub fn urgency_id_medium(mut self, id: String) -> Self {
+    self.urgency_id_medium = Some(id);
+    self
+  }
+
+  /// Set the urgency ID for low-urgency alerts.
+  pub fn urgency_id_low(mut self, id: String) -> Self {
+    self.urgency_id_low = Some(id);
     self
   }
 
@@ -145,7 +169,9 @@ impl RootlyConfigBuilder {
   pub fn done(mut self) -> PagerBuilder {
     self.parent.client_config = Some(PagerClientConfig::Rootly {
       api_key: self.api_key,
-      alert_urgency_id: self.alert_urgency_id,
+      urgency_id_high: self.urgency_id_high,
+      urgency_id_medium: self.urgency_id_medium,
+      urgency_id_low: self.urgency_id_low,
       notification_target_type: self.notification_target_type,
       notification_target_id: self.notification_target_id,
     });

crates/lib/pager/src/client/pager_client.rs

@@ -8,6 +8,7 @@ use rootly_client::requests::create_alert::create_alert::{
 use crate::error::pager_error::PagerError;
 use crate::error::pager_service_error::PagerServiceError;
 use crate::notification::notification_details::NotificationDetails;
+use crate::notification::notification_urgency::NotificationUrgency;
 
 /// The actual client that sends pages.
 #[derive(Clone)]
@@ -23,6 +24,9 @@ pub struct PagerClient {
 
   /// Hostname of the machine sending alerts. Inserted as a label on alerts.
   pub hostname: Option<String>,
+
+  /// Rootly service ID to associate alerts with.
+  pub service_id: Option<String>,
 }
 
 /// Configuration for the pager client backend.
@@ -35,8 +39,14 @@ pub enum PagerClientConfig {
   Rootly {
     api_key: RootlyApiKey,
 
-    /// Alert urgency ID (e.g. "62fde143-..." maps to "High" in our org).
-    alert_urgency_id: Option<String>,
+    /// Alert urgency ID for high-urgency alerts.
+    urgency_id_high: Option<String>,
+
+    /// Alert urgency ID for medium-urgency alerts.
+    urgency_id_medium: Option<String>,
+
+    /// Alert urgency ID for low-urgency alerts.
+    urgency_id_low: Option<String>,
 
     /// Notification target type (e.g. "User", "EscalationPolicy").
     notification_target_type: Option<String>,
@@ -62,8 +72,9 @@ impl PagerClient {
     application_name: Option<String>,
     environment: Option<String>,
     hostname: Option<String>,
+    service_id: Option<String>,
   ) -> Self {
-    Self { client_config, application_name, environment, hostname }
+    Self { client_config, application_name, environment, hostname, service_id }
   }
 
   pub fn is_noop(&self) -> bool {
@@ -89,7 +100,9 @@ impl PagerClient {
   ) -> Result<PageSentResult, PagerError> {
     let PagerClientConfig::Rootly {
       api_key,
-      alert_urgency_id,
+      urgency_id_high,
+      urgency_id_medium,
+      urgency_id_low,
       notification_target_type,
       notification_target_id,
     } = &self.client_config else {
@@ -104,6 +117,14 @@ impl PagerClient {
 
     let mut labels: Vec<(String, String)> = Vec::new();
 
+    if let Some(name) = &self.application_name {
+      labels.push(("application_name".to_string(), name.clone()));
+    }
+
+    if let Some(id) = &self.service_id {
+      labels.push(("service_id".to_string(), id.clone()));
+    }
+
     if let Some(env) = &self.environment {
       labels.push(("environment".to_string(), env.clone()));
     }
@@ -126,11 +147,24 @@ impl PagerClient {
 
     let labels = if labels.is_empty() { None } else { Some(labels) };
 
-    // Enrich the description with HTTP context and hostname if present.
+    // Enrich the description with context and hostname if present.
     let description = match &notification.description {
       Some(desc) => {
         let mut parts = vec![desc.clone()];
 
+        match (&self.application_name, &self.service_id) {
+          (Some(name), Some(id)) => {
+            parts.push(format!("Application: {} (service_id: {})", name, id));
+          }
+          (Some(name), None) => {
+            parts.push(format!("Application: {}", name));
+          }
+          (None, Some(id)) => {
+            parts.push(format!("Service ID: {}", id));
+          }
+          (None, None) => {}
+        }
+
         if let Some(method) = &notification.http_method {
           parts.push(format!("HTTP Method: {}", method));
         }
@@ -152,18 +186,25 @@ impl PagerClient {
       None => None,
     };
 
+    // https://docs.rootly.com/api-reference/alerts/creates-an-alert
     let result = create_alert(CreateAlertArgs {
       api_key: api_key.clone(),
-      source,
+      source: "api".to_string(),
       summary: notification.summary.clone(),
       description,
       status: Some("triggered".to_string()),
-      service_ids: None,
+      service_ids: self.service_id
+          .as_ref()
+          .map(|id| vec![id.clone()]),
       group_ids: None,
       environment_ids: None,
       external_id: None,
       external_url: None,
-      alert_urgency_id: alert_urgency_id.clone(),
+      alert_urgency_id: match notification.urgency.unwrap_or(NotificationUrgency::Medium) {
+        NotificationUrgency::High => urgency_id_high.clone(),
+        NotificationUrgency::Medium => urgency_id_medium.clone(),
+        NotificationUrgency::Low => urgency_id_low.clone(),
+      },
       notification_target_type: notification_target_type.clone(),
       notification_target_id: notification_target_id.clone(),
       labels,

crates/lib/pager/src/notification/generate_deduplication_key.rs

@@ -58,6 +58,7 @@ mod tests {
       http_path: http_path.map(|s| s.to_string()),
       http_status_code: None,
       is_from_error,
+      urgency: None,
     }
   }
 
@@ -212,6 +213,7 @@ mod tests {
         http_path: http_path.map(|s| s.to_string()),
         http_status_code,
         is_from_error,
+        urgency: None,
       }
     }
 

crates/lib/pager/src/notification/mod.rs

@@ -1,3 +1,4 @@
 mod generate_deduplication_key;
 pub mod notification_details;
 pub mod notification_details_builder;
+pub mod notification_urgency;

crates/lib/pager/src/notification/notification_details.rs

@@ -2,6 +2,7 @@ use chrono::{DateTime, Utc};
 use std::fmt::{Debug, Display};
 
 use crate::notification::generate_deduplication_key::generate_deduplication_key;
+use crate::notification::notification_urgency::NotificationUrgency;
 
 /// Details for a pager notification.
 #[derive(Clone, Debug)]
@@ -26,6 +27,9 @@ pub struct NotificationDetails {
 
   /// Whether this notification originated from an error.
   pub is_from_error: bool,
+
+  /// Urgency level for the notification.
+  pub urgency: Option<NotificationUrgency>,
 }
 
 impl NotificationDetails {
@@ -43,6 +47,7 @@ impl NotificationDetails {
       http_path: None,
       http_status_code: None,
       is_from_error: false,
+      urgency: None,
     }
   }
 
@@ -56,6 +61,7 @@ impl NotificationDetails {
       http_path: None,
       http_status_code: None,
       is_from_error: false,
+      urgency: None,
     }
   }
 
@@ -116,6 +122,7 @@ impl NotificationDetails {
       http_path: None,
       http_status_code: None,
       is_from_error: true,
+      urgency: None,
     }
   }
 

crates/lib/pager/src/notification/notification_details_builder.rs

@@ -2,6 +2,7 @@ use chrono::{DateTime, Utc};
 use std::fmt::{Debug, Display};
 
 use crate::notification::notification_details::NotificationDetails;
+use crate::notification::notification_urgency::NotificationUrgency;
 
 pub struct NotificationDetailsBuilder {
   summary: String,
@@ -11,6 +12,7 @@ pub struct NotificationDetailsBuilder {
   http_path: Option<String>,
   http_status_code: Option<u16>,
   is_from_error: bool,
+  urgency: Option<NotificationUrgency>,
 }
 
 impl NotificationDetailsBuilder {
@@ -24,6 +26,7 @@ impl NotificationDetailsBuilder {
       http_path: None,
       http_status_code: None,
       is_from_error: false,
+      urgency: None,
     }
   }
 
@@ -42,8 +45,14 @@ impl NotificationDetailsBuilder {
       http_path: None,
       http_status_code: None,
       is_from_error: true,
+      urgency: None,
     }
   }
+  
+  pub fn set_summary(mut self, summary: String) -> Self {
+    self.summary = summary;
+    self
+  }
 
   pub fn set_description(mut self, description: Option<String>) -> Self {
     self.description = description;
@@ -65,6 +74,11 @@ impl NotificationDetailsBuilder {
     self
   }
 
+  pub fn set_urgency(mut self, urgency: Option<NotificationUrgency>) -> Self {
+    self.urgency = urgency;
+    self
+  }
+
   pub fn build(self) -> NotificationDetails {
     NotificationDetails {
       summary: self.summary,
@@ -74,6 +88,7 @@ impl NotificationDetailsBuilder {
       http_path: self.http_path,
       http_status_code: self.http_status_code,
       is_from_error: self.is_from_error,
+      urgency: self.urgency,
     }
   }
 }

crates/lib/pager/src/notification/notification_urgency.rs

@@ -0,0 +1,7 @@
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum NotificationUrgency {
+  High,
+  Medium,
+  Low,
+}

crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs

@@ -46,7 +46,8 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken;
 use tokens::tokens::media_files::MediaFileToken;
 use url::Url;
 use url_utils::extension::extract_extension_from_url::{extract_extension_from_url, ExtractExtensions};
-use pager::notification::notification_details::NotificationDetails;
+use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
 use crate::http_server::session::lookup::user_session_feature_flags::UserSessionFeatureFlags;
 
 // ======================== Result of a successful generation ========================
@@ -221,7 +222,10 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
           Err(err) => {
             warn!("Regular session fallback also failed for user {:?}: {:?}", user_token, err);
             if let Err(page_err) = server_state.pager.enqueue_page(
-              NotificationDetails::from_error_with_context("Seedance 2.0 generation failed (whitelist + fallback)", &err)
+              NotificationDetailsBuilder::from_error(&err)
+                  .set_summary("Seedance 2.0 generation failed (whitelist + fallback)".to_string())
+                  .set_urgency(Some(NotificationUrgency::High))
+                  .build()
             ) {
               warn!("Failed to enqueue pager alert: {:?}", page_err);
             }
@@ -252,7 +256,10 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
       Err(err) => {
         warn!("Error calling seedance2pro generate_video: {:?}", err);
         if let Err(page_err) = server_state.pager.enqueue_page(
-          NotificationDetails::from_error_with_context("Seedance 2.0 generation failed", &err)
+          NotificationDetailsBuilder::from_error(&err)
+              .set_summary("Seedance 2.0 generation failed".to_string())
+              .set_urgency(Some(NotificationUrgency::High))
+              .build()
         ) {
           warn!("Failed to enqueue pager alert: {:?}", page_err);
         }

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_common_web_error.rs

@@ -2,6 +2,7 @@ use log::{debug, warn};
 
 use pager::client::pager::Pager;
 use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
 
 use crate::http_server::common_responses::common_web_error::CommonWebError;
 
@@ -30,6 +31,7 @@ pub(super) fn check_common_web_error(
           .set_http_method(Some(method.to_string()))
           .set_http_path(Some(path.to_string()))
           .set_http_status_code(Some(500))
+          .set_urgency(Some(NotificationUrgency::Medium))
           .build();
 
       if let Err(err) = pager.enqueue_page(notification) {

crates/service/web/storyteller_web/src/http_server/middleware/error_alerting_middleware/check_status_code_fallback.rs

@@ -2,6 +2,7 @@ use log::{debug, warn};
 
 use pager::client::pager::Pager;
 use pager::notification::notification_details_builder::NotificationDetailsBuilder;
+use pager::notification::notification_urgency::NotificationUrgency;
 
 /// Fallback alerting based on HTTP status code when no typed error matched.
 pub(super) fn check_status_code_fallback(
@@ -26,6 +27,7 @@ pub(super) fn check_status_code_fallback(
           .set_http_method(Some(method.to_string()))
           .set_http_path(Some(path.to_string()))
           .set_http_status_code(Some(status_code))
+          .set_urgency(Some(NotificationUrgency::Medium))
           .build();
 
       if let Err(err) = pager.enqueue_page(notification) {

crates/service/web/storyteller_web/src/startup/build_pager.rs

@@ -6,6 +6,13 @@ use rootly_client::creds::rootly_api_key::RootlyApiKey;
 
 use crate::state::flags::paging_flags::PagingFlags;
 
+/// `storyteller-web` rootly service ID
+const ROOTLY_SERVICE_ID: &str = "9ebebb09-0c56-4c8a-8f57-d5f0f85f3f16";
+
+const ROOTLY_URGENCY_ID_HIGH: &str = "62fde143-1258-4639-9ee6-1400ebf7308d";
+const ROOTLY_URGENCY_ID_MEDIUM: &str = "7366ba5e-f6ea-4a4e-a1b4-ae43d512e1e2";
+const ROOTLY_URGENCY_ID_LOW: &str = "4db3d3ed-f4ed-4818-82f5-7746da404bd2";
+
 pub fn build_pager(
   server_environment: server_environment::ServerEnvironment,
   hostname: &str,
@@ -23,7 +30,8 @@ pub fn build_pager(
   let builder = PagerBuilder::new()
       .application_name("storyteller-web".to_string())
       .environment(environment.to_string())
-      .hostname(hostname.to_string());
+      .hostname(hostname.to_string())
+      .service_id(ROOTLY_SERVICE_ID.to_string());
   
   // If paging is globally disabled, use a NoOp pager regardless of API key.
   if !paging_flags.is_paging_enabled {
@@ -50,11 +58,10 @@ pub fn build_pager(
 
 fn build_rootly_pager(builder: PagerBuilder, api_key: String) -> (Pager, PagerWorker) {
   let mut rootly_builder = builder
-      .rootly(RootlyApiKey::new(api_key));
-
-  if let Some(urgency_id) = easyenv::get_env_string_optional("ROOTLY_ALERT_URGENCY_ID") {
-    rootly_builder = rootly_builder.alert_urgency_id(urgency_id);
-  }
+      .rootly(RootlyApiKey::new(api_key))
+      .urgency_id_high(ROOTLY_URGENCY_ID_HIGH.to_string())
+      .urgency_id_medium(ROOTLY_URGENCY_ID_MEDIUM.to_string())
+      .urgency_id_low(ROOTLY_URGENCY_ID_LOW.to_string());
 
   let target_type = easyenv::get_env_string_optional("ROOTLY_NOTIFICATION_TARGET_TYPE");
   let target_id = easyenv::get_env_string_optional("ROOTLY_NOTIFICATION_TARGET_ID");