alert on seedance failures

Cargo.lock

@@ -11263,6 +11263,7 @@ dependencies = [
  "mimetypes",
  "mysql_queries",
  "once_cell",
+ "pager",
  "password",
  "primitives",
  "prompt_engineering",
@@ -11275,6 +11276,7 @@ dependencies = [
  "regex",
  "resend-rs",
  "reusable_types",
+ "rootly_client",
  "rust-s3",
  "rustls 0.23.36",
  "seedance2pro_client",

crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/mod.rs

@@ -0,0 +1 @@
+pub mod moderation_send_alert;

crates/api_clients/artcraft/artcraft_api_defs/src/moderation/alerts/moderation_send_alert.rs

@@ -0,0 +1,18 @@
+use serde_derive::{Deserialize, Serialize};
+use utoipa::ToSchema;
+
+pub const MODERATION_SEND_ALERT_PATH: &str = "/v1/moderation/alerts/send";
+
+#[derive(Deserialize, ToSchema)]
+pub struct ModerationSendAlertRequest {
+  /// Optional title for the alert. Defaults to "Test Moderation Alert".
+  pub title: Option<String>,
+
+  /// Optional description for the alert. Defaults to "This is a test moderation alert."
+  pub description: Option<String>,
+}
+
+#[derive(Serialize, ToSchema)]
+pub struct ModerationSendAlertResponse {
+  pub success: bool,
+}

crates/api_clients/artcraft/artcraft_api_defs/src/moderation/mod.rs

@@ -1,3 +1,4 @@
+pub mod alerts;
 pub mod jobs;
 pub mod user;
 pub mod wallet_ledger_entries;

crates/lib/pager/src/client/pager.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 
-use log::info;
+use log::{info, warn};
 
 use crate::client::pager_client::{PageSentResult, PagerClient};
 use crate::error::pager_error::PagerError;
@@ -15,20 +15,9 @@ use crate::worker::pager_worker_message_queue::PagerWorkerMessageQueue;
 /// - **Immediate**: `send_page_immediately()` sends inline (blocks until API responds).
 /// - **Queued**: `enqueue_page()` pushes to a background worker (non-blocking).
 ///
-/// Build an instance via `PagerBuilder`:
-/// ```ignore
-/// // Without worker:
-/// let pager = PagerBuilder::new()
-///     .application_name("my-service".to_string())
-///     .rootly(api_key)
-///     .build()?;
+/// Build an instance via `PagerBuilder`.
 ///
-/// // With worker:
-/// let (pager, worker) = PagerBuilder::new()
-///     .application_name("my-service".to_string())
-///     .rootly(api_key)
-///     .build_with_worker()?;
-/// ```
+#[derive(Clone)]
 pub struct Pager {
   client: PagerClient,
   queue: Option<Arc<PagerWorkerMessageQueue>>,
@@ -62,6 +51,16 @@ impl Pager {
     self.client.send_page(&notification).await
   }
 
+  /// Send a page immediately, blocking until the API responds.
+  pub async fn send_page_immediately_infallible(
+    &self,
+    notification: NotificationDetails,
+  ) {
+    if let Err(err) = self.send_page_immediately(notification).await {
+      warn!("Failure sending page: {:?}", err);
+    }
+  }
+
   /// Enqueue a page to be sent by the background worker thread.
   ///
   /// Returns an error if the worker is not configured (use `build_with_worker()`).

crates/schema/database/mysql_queries/src/queries/users/user_sessions/get_user_session_by_token.rs

@@ -13,6 +13,8 @@ use tokens::tokens::users::UserToken;
 
 use crate::helpers::boolean_converters::{i8_to_bool, nullable_i8_to_bool_default_false, nullable_i8_to_optional_bool};
 
+const ADMIN_ROLE_SLUG: &str = "admin";
+
 #[derive(Clone, Serialize, Deserialize)]
 pub struct SessionUserRecord {
   pub user_token: UserToken,
@@ -90,7 +92,7 @@ impl SessionUserRecord {
   }
 
   pub fn is_mod(&self) -> bool {
-    self.can_ban_users
+    self.can_ban_users || self.user_role_slug == ADMIN_ROLE_SLUG
   }
 }
 

crates/service/web/storyteller_web/Cargo.toml

@@ -65,7 +65,9 @@ memory_caching = { path = "../../../lib/caching/memory_caching" }
 migration = { path = "../../../schema/database/migration" }
 mimetypes.workspace = true
 mysql_queries.workspace = true
+pager.workspace = true
 password = { path = "../../../lib/password" }
+rootly_client.workspace = true
 primitives = { path = "../../../lib/data/primitives" }
 prompt_engineering = { path = "../../../lib/deprecated/machine_learning/prompt_engineering" }
 redis_caching = { path = "../../../lib/caching/redis_caching" }

crates/service/web/storyteller_web/src/api_doc_cli.rs

@@ -32,6 +32,7 @@ pub mod docs;
 pub mod email;
 pub mod error;
 pub mod http_server;
+pub mod startup;
 pub mod state;
 pub mod threads;
 pub mod util;

crates/service/web/storyteller_web/src/api_doc_server.rs

@@ -34,6 +34,7 @@ pub mod docs;
 pub mod email;
 pub mod error;
 pub mod http_server;
+pub mod startup;
 pub mod state;
 pub mod threads;
 pub mod util;

crates/service/web/storyteller_web/src/docs/api_doc.rs

@@ -278,6 +278,7 @@ use artcraft_api_defs::subscriptions::get_session_subscription::*;
 use artcraft_api_defs::media_file::list_batch_generated_media_files::*;
 // Handler modules with locally-defined types
 use crate::http_server::endpoints::moderation::info::moderator_token_info_handler::*;
+use artcraft_api_defs::moderation::alerts::moderation_send_alert::*;
 use artcraft_api_defs::moderation::user::list_subscribing_users_by_signup_date::*;
 use artcraft_api_defs::moderation::user::list_users_by_signup_date::*;
 use artcraft_api_defs::moderation::user::user_lookup::*;
@@ -467,6 +468,7 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
     // TTS
     crate::http_server::endpoints::tts::enqueue_infer_f5_tts_handler::enqueue_infer_f5_tts_handler::enqueue_infer_f5_tts_handler,
     // Moderation
+    crate::http_server::endpoints::moderation::alerts::moderation_send_alert_handler::moderation_send_alert_handler,
     crate::http_server::endpoints::moderation::info::moderator_token_info_handler::moderator_get_token_info_handler,
     crate::http_server::endpoints::moderation::user::moderator_list_subscribing_users_by_signup_date::moderator_list_subscribing_users_by_signup_date_handler,
     crate::http_server::endpoints::moderation::user::moderator_list_users_by_signup_date::moderator_list_users_by_signup_date_handler,
@@ -661,6 +663,8 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
     EditUserFeatureFlagsError,
     EditUserFeatureFlagsOption,
     EditUserFeatureFlagsRequest,
+    ModerationSendAlertRequest,
+    ModerationSendAlertResponse,
     ModeratorListSubscribingUsersBySignupDateRequest,
     ModeratorListSubscribingUsersBySignupDateResponse,
     ModeratorListSubscribingUsersBySignupDateEntry,

crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/seedance_2p0_multi_function_video_gen_handler.rs

@@ -46,6 +46,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken;
 use tokens::tokens::media_files::MediaFileToken;
 use url::Url;
 use url_utils::extension::extract_extension_from_url::{extract_extension_from_url, ExtractExtensions};
+use pager::notification::notification_details::NotificationDetails;
 use crate::http_server::session::lookup::user_session_feature_flags::UserSessionFeatureFlags;
 
 // ======================== Result of a successful generation ========================
@@ -219,6 +220,11 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
           }
           Err(err) => {
             warn!("Regular session fallback also failed for user {:?}: {:?}", user_token, err);
+            if let Err(page_err) = server_state.pager.enqueue_page(
+              NotificationDetails::from_error_with_context("Seedance 2.0 generation failed (whitelist + fallback)", &err)
+            ) {
+              warn!("Failed to enqueue pager alert: {:?}", page_err);
+            }
             refund_wallet_after_api_failure(&deduction_result.ledger_entry_token, &mut mysql_connection).await?;
             return Err(CommonWebError::ServerError);
           }
@@ -245,6 +251,11 @@ pub async fn seedance_2p0_multi_function_video_gen_handler(
       Ok(result) => result,
       Err(err) => {
         warn!("Error calling seedance2pro generate_video: {:?}", err);
+        if let Err(page_err) = server_state.pager.enqueue_page(
+          NotificationDetails::from_error_with_context("Seedance 2.0 generation failed", &err)
+        ) {
+          warn!("Failed to enqueue pager alert: {:?}", page_err);
+        }
         refund_wallet_after_api_failure(&deduction_result.ledger_entry_token, &mut mysql_connection).await?;
         return Err(CommonWebError::ServerError);
       }

crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/mod.rs

@@ -0,0 +1 @@
+pub mod moderation_send_alert_handler;

crates/service/web/storyteller_web/src/http_server/endpoints/moderation/alerts/moderation_send_alert_handler.rs

@@ -0,0 +1,58 @@
+use std::sync::Arc;
+
+use actix_web::web::Json;
+use actix_web::{web, HttpRequest};
+use log::warn;
+
+use artcraft_api_defs::moderation::alerts::moderation_send_alert::{
+  ModerationSendAlertRequest,
+  ModerationSendAlertResponse,
+};
+use pager::client::pager::Pager;
+use pager::notification::notification_details::NotificationDetails;
+
+use crate::http_server::common_responses::common_web_error::CommonWebError;
+use crate::http_server::web_utils::user_session::require_moderator::{require_moderator, UseDatabase};
+use crate::state::server_state::ServerState;
+
+/// Send a test alert via the pager system. Moderators only.
+#[utoipa::path(
+  post,
+  tag = "Moderation",
+  path = "/v1/moderation/alerts/send",
+  request_body = ModerationSendAlertRequest,
+  responses(
+    (status = 200, description = "Success", body = ModerationSendAlertResponse),
+    (status = 401, description = "Unauthorized"),
+    (status = 500, description = "Server error"),
+  ),
+)]
+pub async fn moderation_send_alert_handler(
+  http_request: HttpRequest,
+  request: Json<ModerationSendAlertRequest>,
+  server_state: web::Data<Arc<ServerState>>,
+) -> Result<Json<ModerationSendAlertResponse>, CommonWebError> {
+
+  let _user_session = require_moderator(&http_request, &server_state, UseDatabase::GrabNewConnection)
+    .await
+    .map_err(|_| CommonWebError::NotAuthorized)?;
+
+  let title = request.title.clone()
+    .unwrap_or_else(|| "Test Moderation Alert".to_string());
+
+  let description = request.description.clone()
+    .unwrap_or_else(|| "This is a test moderation alert.".to_string());
+
+  let notification = NotificationDetails::with_summary_and_description(title, description);
+
+  server_state.pager
+    .enqueue_page(notification)
+    .map_err(|err| {
+      warn!("moderation_send_alert error: {:?}", err);
+      CommonWebError::ServerError
+    })?;
+
+  Ok(Json(ModerationSendAlertResponse {
+    success: true,
+  }))
+}

crates/service/web/storyteller_web/src/http_server/endpoints/moderation/mod.rs

@@ -1,3 +1,4 @@
+pub mod alerts;
 pub mod info;
 pub mod ip_bans;
 pub mod jobs;

crates/service/web/storyteller_web/src/http_server/routes/application_routes/moderation_routes.rs

@@ -35,6 +35,7 @@ use crate::http_server::endpoints::moderation::user::moderator_list_subscribing_
 use crate::http_server::endpoints::moderation::user::moderator_list_users_by_signup_date::moderator_list_users_by_signup_date_handler;
 use crate::http_server::endpoints::moderation::user::moderator_user_lookup_by_stripe_customer_id_handler::moderator_user_lookup_by_stripe_customer_id_handler;
 use crate::http_server::endpoints::moderation::user::moderator_user_lookup_handler::moderator_user_lookup_handler;
+use crate::http_server::endpoints::moderation::alerts::moderation_send_alert_handler::moderation_send_alert_handler;
 use crate::http_server::endpoints::moderation::user_feature_flags::edit_user_feature_flags_handler::edit_user_feature_flags_handler;
 
 pub fn add_moderator_routes<T, B> (app: App<T>) -> App<T>
@@ -49,6 +50,12 @@ pub fn add_moderator_routes<T, B> (app: App<T>) -> App<T>
       >,
 {
   app.service(web::scope("/v1/moderation")
+        .service(web::scope("/alerts")
+            .service(web::resource("/send")
+                .route(web::post().to(moderation_send_alert_handler))
+                .route(web::head().to(|| HttpResponse::Ok()))
+            )
+        )
         .service(web::resource("/user_feature_flags/{username_or_token}")
             .route(web::post().to(edit_user_feature_flags_handler))
             .route(web::head().to(|| HttpResponse::Ok()))

crates/service/web/storyteller_web/src/http_server/web_utils/user_session/require_moderator.rs

@@ -75,7 +75,7 @@ pub async fn require_moderator(
     }
   };
 
-  if !user_session.can_ban_users {
+  if !user_session.is_mod() {
     warn!("user is not a moderator: {:?}", user_session.user_token.as_str());
     return Err(RequireModeratorError::NotAuthorized);
   }

crates/service/web/storyteller_web/src/main.rs

@@ -52,7 +52,7 @@ use elasticsearch::http::transport::Transport;
 use elasticsearch::Elasticsearch;
 use errors::AnyhowResult;
 use fal_client::creds::fal_api_key::FalApiKey;
-use log::info;
+use log::{info, warn};
 use memory_caching::arc_ttl_sieve::ArcTtlSieve;
 use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
 use mysql_queries::mediators::badge_granter::BadgeGranter;
@@ -79,6 +79,7 @@ use crate::http_server::web_utils::handle_multipart_error::handle_multipart_erro
 use crate::http_server::web_utils::scoped_temp_dir_creator::ScopedTempDirCreator;
 use crate::state::certs::google_sign_in_cert::GoogleSignInCert;
 use crate::state::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache;
+use crate::startup::build_pager::build_pager;
 use crate::state::server_state::{DurableInMemoryCaches, EnvConfig, EphemeralInMemoryCaches, FalData, InMemoryCaches, OpenAiData, ResendData, Seedance2ProData, ServerInfo, ServerState, StaticFeatureFlags, StripeSettings, TrollBans, WorldLabsData};
 use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus;
 use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread;
@@ -93,6 +94,7 @@ pub mod configs;
 pub mod email;
 pub mod error;
 pub mod http_server;
+pub mod startup;
 pub mod state;
 pub mod threads;
 pub mod util;
@@ -328,6 +330,21 @@ async fn main() -> AnyhowResult<()> {
     poll_model_token_info_thread(model_token_info_cache2, mysql_pool5).await;
   });
 
+  let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?)
+      .ok_or(anyhow!("invalid server environment"))?;
+
+  let server_environment_typed = match server_environment {
+    ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
+    ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
+  };
+
+  let (pager, pager_worker) = build_pager(server_environment_typed);
+
+  info!("Spawning pager worker thread.");
+
+  tokio_runtime.spawn(async move {
+    pager_worker.run().await;
+  });
 
   let stripe_configs = StripeConfig {
     checkout: StripeCheckoutConfigs {
@@ -345,9 +362,6 @@ async fn main() -> AnyhowResult<()> {
     },
   };
 
-  let server_environment = ServerEnvironment::from_str(&easyenv::get_env_string_required("SERVER_ENVIRONMENT")?)
-      .ok_or(anyhow!("invalid server environment"))?;
-
   let service_feature_flags = StaticFeatureFlags {
     // Permanent (control plane / safety) flags : messaging
     maybe_status_alert_category: easyenv::get_env_string_optional("FF_STATUS_ALERT_CATEGORY"),
@@ -391,11 +405,6 @@ async fn main() -> AnyhowResult<()> {
       .unwrap_or(String::from("unknown"))
       .trim()
       .to_string();
-
-  let server_environment_typed = match server_environment {
-    ServerEnvironment::Production => server_environment::ServerEnvironment::Production,
-    ServerEnvironment::Development => server_environment::ServerEnvironment::Development,
-  };
   
   let fal_api_key = FalApiKey::new(easyenv::get_env_string_required("FAL_API_KEY")?);
   let fal_webhook_url = easyenv::get_env_string_required("FAL_WEBHOOK_URL")?;
@@ -473,6 +482,7 @@ async fn main() -> AnyhowResult<()> {
     worldlabs: WorldLabsData {
       api_key: worldlabs_api_key,
     },
+    pager,
     caches: InMemoryCaches {
       durable: DurableInMemoryCaches {
         model_token_info: model_token_info_cache,
@@ -652,6 +662,7 @@ pub async fn serve(server_state: ServerState) -> AnyhowResult<()>
       .app_data(web::Data::new(server_state_arc.stripe.clone().client.clone()))
       .app_data(web::Data::new(server_state_arc.third_party_url_redirector))
       .app_data(web::Data::new(server_state_arc.google_sign_in_cert.clone()))
+      .app_data(web::Data::new(server_state_arc.pager.clone()))
       .app_data(web::Data::new(old_server_environment))
       .app_data(web::Data::new(new_server_environment))
       .app_data(web::Data::from(product_lookup)) // NB: Data::from(Arc<T>) for dynamic dispatch

crates/service/web/storyteller_web/src/startup/build_pager.rs

@@ -0,0 +1,61 @@
+use log::{info, warn};
+use pager::client::pager::Pager;
+use pager::client::pager_builder::PagerBuilder;
+use pager::worker::pager_worker::PagerWorker;
+use rootly_client::creds::rootly_api_key::RootlyApiKey;
+
+pub fn build_pager(server_environment: server_environment::ServerEnvironment) -> (Pager, PagerWorker) {
+  let environment = if server_environment.is_deployed_in_production() {
+    "production"
+  } else {
+    "development"
+  };
+
+  let maybe_api_key = easyenv::get_env_string_optional("ROOTLY_API_KEY");
+
+  match maybe_api_key {
+    Some(api_key) => {
+      info!("Rootly API key found. Configuring pager with Rootly backend.");
+      build_rootly_pager(environment, api_key)
+    }
+    None => {
+      warn!("ROOTLY_API_KEY not set. Pager will not send real pages.");
+      build_fallback_pager(environment)
+    }
+  }
+}
+
+fn build_rootly_pager(environment: &str, api_key: String) -> (Pager, PagerWorker) {
+  let mut rootly_builder = PagerBuilder::new()
+      .application_name("storyteller-web".to_string())
+      .environment(environment.to_string())
+      .rootly(RootlyApiKey::new(api_key));
+
+  if let Some(urgency_id) = easyenv::get_env_string_optional("ROOTLY_ALERT_URGENCY_ID") {
+    rootly_builder = rootly_builder.alert_urgency_id(urgency_id);
+  }
+
+  let target_type = easyenv::get_env_string_optional("ROOTLY_NOTIFICATION_TARGET_TYPE");
+  let target_id = easyenv::get_env_string_optional("ROOTLY_NOTIFICATION_TARGET_ID");
+
+  if let (Some(t_type), Some(t_id)) = (target_type, target_id) {
+    rootly_builder = rootly_builder.notification_target(t_type, t_id);
+  }
+
+  match rootly_builder.build_with_worker() {
+    Ok((pager, worker)) => (pager, worker),
+    Err(err) => {
+      warn!("Failed to build pager with Rootly backend: {}. Pages will not be sent.", err);
+      build_fallback_pager(environment)
+    }
+  }
+}
+
+fn build_fallback_pager(environment: &str) -> (Pager, PagerWorker) {
+  PagerBuilder::new()
+    .application_name("storyteller-web".to_string())
+    .environment(environment.to_string())
+    .rootly(RootlyApiKey::new(String::new()))
+    .build_with_worker()
+    .expect("fallback pager build should not fail")
+}

crates/service/web/storyteller_web/src/startup/mod.rs

@@ -0,0 +1 @@
+pub mod build_pager;

crates/service/web/storyteller_web/src/state/server_state.rs

@@ -9,6 +9,7 @@ use elasticsearch::Elasticsearch;
 use fal_client::creds::fal_api_key::FalApiKey;
 use memory_caching::arc_ttl_sieve::ArcTtlSieve;
 use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
+use pager::client::pager::Pager;
 use mysql_queries::mediators::badge_granter::BadgeGranter;
 use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
 use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult;
@@ -100,6 +101,8 @@ pub struct ServerState {
 
   pub worldlabs: WorldLabsData,
 
+  pub pager: Pager,
+
   /// Where to store audio uploads for w2l
   pub audio_uploads_bucket_root: String,