improve metrics and report breakage

_metrics/datadog/dashboards/storyteller_web_http_overview.json

@@ -1,5 +1,5 @@
 {
-  "title": "Storyteller-web — HTTP overview",
+  "title": "Storyteller-web \u2014 HTTP overview",
   "description": "Per-request HTTP metrics emitted by the storyteller-web metrics middleware (`http.server.request.count` counter, `http.server.request.duration_ms` distribution). Filter by `env` / `service` / `route` using the template variables at the top.\n\nMetrics emitted from `crates/lib/metrics` + `crates/service/web/storyteller_web/src/http_server/middleware/metrics_middleware/`. Source of truth for this dashboard JSON is `_metrics/datadog/dashboards/storyteller_web_http_overview.json` in the storyteller monorepo. Apply changes with `script/metrics/apply_dashboards.sh`.",
   "layout_type": "ordered",
   "notify_list": [],
@@ -105,7 +105,7 @@
               "custom_unit": "ms",
               "requests": [
                 {
-                  "q": "avg:http.server.request.duration_ms.p95{$env,$service,$route}",
+                  "q": "p95:http.server.request.duration_ms{$env,$service,$route}",
                   "aggregator": "avg",
                   "conditional_formats": [
                     {
@@ -145,7 +145,7 @@
               "custom_unit": "ms",
               "requests": [
                 {
-                  "q": "avg:http.server.request.duration_ms.p99{$env,$service,$route}",
+                  "q": "p99:http.server.request.duration_ms{$env,$service,$route}",
                   "aggregator": "avg"
                 }
               ]
@@ -220,7 +220,7 @@
               "legend_layout": "horizontal",
               "requests": [
                 {
-                  "q": "avg:http.server.request.duration_ms.p50{$env,$service,$route}",
+                  "q": "p50:http.server.request.duration_ms{$env,$service,$route}",
                   "display_type": "line",
                   "style": {
                     "palette": "cool",
@@ -229,7 +229,7 @@
                   }
                 },
                 {
-                  "q": "avg:http.server.request.duration_ms.p95{$env,$service,$route}",
+                  "q": "p95:http.server.request.duration_ms{$env,$service,$route}",
                   "display_type": "line",
                   "style": {
                     "palette": "warm",
@@ -238,7 +238,7 @@
                   }
                 },
                 {
-                  "q": "avg:http.server.request.duration_ms.p99{$env,$service,$route}",
+                  "q": "p99:http.server.request.duration_ms{$env,$service,$route}",
                   "display_type": "line",
                   "style": {
                     "palette": "warm",
@@ -303,7 +303,7 @@
               "title_align": "left",
               "requests": [
                 {
-                  "q": "top(avg:http.server.request.duration_ms.p95{$env,$service,$route} by {route}, 10, 'mean', 'desc')"
+                  "q": "top(p95:http.server.request.duration_ms{$env,$service,$route} by {route}, 10, 'mean', 'desc')"
                 }
               ]
             },
@@ -431,7 +431,102 @@
         "width": 12,
         "height": 5
       }
+    },
+    {
+      "definition": {
+        "type": "group",
+        "layout_type": "ordered",
+        "title": "Metrics worker health",
+        "show_title": true,
+        "widgets": [
+          {
+            "definition": {
+              "type": "query_value",
+              "title": "Startup pings (last 30m)",
+              "title_size": "16",
+              "title_align": "left",
+              "precision": 0,
+              "autoscale": true,
+              "requests": [
+                {
+                  "q": "sum:metrics.startup.ping{$env,$service}.as_count()",
+                  "aggregator": "sum",
+                  "conditional_formats": [
+                    {
+                      "comparator": "<=",
+                      "value": 0,
+                      "palette": "white_on_red"
+                    },
+                    {
+                      "comparator": ">",
+                      "value": 0,
+                      "palette": "white_on_green"
+                    }
+                  ]
+                }
+              ]
+            },
+            "layout": {
+              "x": 0,
+              "y": 0,
+              "width": 4,
+              "height": 2
+            }
+          },
+          {
+            "definition": {
+              "type": "toplist",
+              "title": "Hosts with recent startup pings",
+              "title_size": "16",
+              "title_align": "left",
+              "requests": [
+                {
+                  "q": "top(sum:metrics.startup.ping{$env,$service} by {host}.as_count(), 25, 'sum', 'desc')"
+                }
+              ]
+            },
+            "layout": {
+              "x": 4,
+              "y": 0,
+              "width": 4,
+              "height": 4
+            }
+          },
+          {
+            "definition": {
+              "type": "timeseries",
+              "title": "Startup pings over time (one bar per host restart)",
+              "title_size": "16",
+              "title_align": "left",
+              "show_legend": true,
+              "legend_layout": "horizontal",
+              "requests": [
+                {
+                  "q": "sum:metrics.startup.ping{$env,$service} by {host}.as_count()",
+                  "display_type": "bars"
+                }
+              ],
+              "yaxis": {
+                "scale": "linear",
+                "include_zero": true
+              }
+            },
+            "layout": {
+              "x": 8,
+              "y": 0,
+              "width": 4,
+              "height": 4
+            }
+          }
+        ]
+      },
+      "layout": {
+        "x": 0,
+        "y": 26,
+        "width": 12,
+        "height": 4
+      }
     }
   ],
   "id": "thp-c27-vnv"
-}
+}

crates/lib/metrics/src/worker.rs

@@ -3,7 +3,9 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Duration;
 
-use log::{debug, info, warn};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use log::{error, info, warn};
 
 use datadog_client::client::{
   DatadogClient, DistributionPoint, DistributionSeries, MetricPoint, MetricSeries,
@@ -48,10 +50,18 @@ impl MetricsWorker {
 
   pub async fn run(&self) {
     info!(
-      "Metrics worker started (flush every {:?}, service={}, env={})",
-      self.config.flush_interval, self.config.service_name, self.config.env,
+      "Metrics worker started (flush every {:?}, service={}, env={}, host={})",
+      self.config.flush_interval,
+      self.config.service_name,
+      self.config.env,
+      self.config.host.as_deref().unwrap_or("<unset>"),
     );
 
+    // Startup connectivity ping: submit one tiny counter so logs immediately
+    // reveal misconfigured API keys / blocked egress / etc., rather than
+    // staying silent until the first batch with real data flushes.
+    self.startup_ping().await;
+
     let mut ticker = tokio::time::interval(self.config.flush_interval);
     // `tokio::time::interval` fires immediately on the first tick; skip it
     // so we don't flush a half-empty bucket the moment we boot.
@@ -68,6 +78,39 @@ impl MetricsWorker {
     info!("Metrics worker stopped");
   }
 
+  /// One-shot at worker startup. Submits a single `metrics.startup.ping`
+  /// counter so failures (bad API key, no egress, wrong DD site) surface
+  /// in the logs within seconds of boot.
+  async fn startup_ping(&self) {
+    let now = SystemTime::now()
+      .duration_since(UNIX_EPOCH)
+      .map(|d| d.as_secs() as i64)
+      .unwrap_or(0);
+    let ping = MetricSeries {
+      metric: "metrics.startup.ping".to_string(),
+      points: vec![MetricPoint(now, 1.0)],
+      metric_type: "count".to_string(),
+      tags: vec![
+        format!("env:{}", self.config.env),
+        format!("service:{}", self.config.service_name),
+      ],
+      host: self.config.host.clone(),
+      interval: Some(60),
+    };
+    match self.client.submit_series(vec![ping]).await {
+      Ok(()) => info!(
+        "metrics: startup ping submitted successfully (service={}, env={})",
+        self.config.service_name, self.config.env,
+      ),
+      Err(e) => error!(
+        "metrics: startup ping FAILED ({}). The worker will keep trying, but \
+         this almost always means a misconfigured DATADOG_API_KEY, a wrong \
+         Datadog site host, or blocked egress to api.datadoghq.com.",
+        e,
+      ),
+    }
+  }
+
   async fn flush_once(&self) {
     let samples = self.queue.drain();
     if samples.is_empty() {
@@ -75,16 +118,33 @@ impl MetricsWorker {
     }
     let sample_count = samples.len();
     let MetricsBuild { distributions, counts } = self.build_metrics(samples);
-    debug!(
-      "metrics flush: {} samples → {} distribution series, {} count series",
-      sample_count, distributions.len(), counts.len(),
-    );
+    let dist_count = distributions.len();
+    let count_count = counts.len();
 
-    if let Err(e) = self.client.submit_distribution_points(distributions).await {
-      warn!("metrics: distribution submission failed: {}", e);
-    }
-    if let Err(e) = self.client.submit_series(counts).await {
-      warn!("metrics: series submission failed: {}", e);
+    let dist_result = self.client.submit_distribution_points(distributions).await;
+    let series_result = self.client.submit_series(counts).await;
+
+    match (&dist_result, &series_result) {
+      (Ok(()), Ok(())) => {
+        info!(
+          "metrics flush ok: {} samples → {} distribution series, {} count series",
+          sample_count, dist_count, count_count,
+        );
+      }
+      _ => {
+        if let Err(e) = dist_result {
+          warn!(
+            "metrics: distribution submission FAILED ({} series, {} samples): {}",
+            dist_count, sample_count, e,
+          );
+        }
+        if let Err(e) = series_result {
+          warn!(
+            "metrics: series submission FAILED ({} series, {} samples): {}",
+            count_count, sample_count, e,
+          );
+        }
+      }
     }
   }
 

crates/service/web/storyteller_web/src/startup/setup_metrics.rs

@@ -16,22 +16,33 @@ pub fn build_metrics(
   server_environment: server_environment::ServerEnvironment,
   hostname: &str,
 ) -> (MetricsCollector, Option<MetricsWorker>) {
-  if !env_enable_metrics_default_false() {
-    info!("ENABLE_METRICS=false (or unset). Metrics collection is disabled.");
-    return (MetricsCollector::noop(), None);
-  }
-
-  let Some(api_key) = env_optional_datadog_api_key() else {
-    warn!("ENABLE_METRICS=true but DATADOG_API_KEY is not set. Metrics will be a no-op.");
-    return (MetricsCollector::noop(), None);
-  };
-
+  // Loud-on-purpose: silence in the logs after deploy is the most common
+  // way "metrics aren't showing up" goes unnoticed. Make the disabled-state
+  // unmistakable in every env.
   let env_label = if server_environment.is_deployed_in_production() {
     "production"
   } else {
     "development"
   };
 
+  if !env_enable_metrics_default_false() {
+    warn!(
+      "METRICS DISABLED: ENABLE_METRICS=false (or unset) — no Datadog metrics \
+       will be reported (env={}, host={}). Set ENABLE_METRICS=true to enable.",
+      env_label, hostname,
+    );
+    return (MetricsCollector::noop(), None);
+  }
+
+  let Some(api_key) = env_optional_datadog_api_key() else {
+    warn!(
+      "METRICS DISABLED: ENABLE_METRICS=true but DATADOG_API_KEY is not set \
+       (env={}, host={}). Set DATADOG_API_KEY to enable submission.",
+      env_label, hostname,
+    );
+    return (MetricsCollector::noop(), None);
+  };
+
   let client = DatadogClient::new(DatadogApiKey::new(api_key));
 
   let build = MetricsBuilder::new()