mirror of
https://github.com/storytold/artcraft.git
synced 2026-10-20 17:02:53 -04:00
metrics defs (#1554)
This commit is contained in:
Brandon Thomas
@@ -16,6 +16,7 @@ env-secrets
|
|||||||
pubkeys.txt
|
pubkeys.txt
|
||||||
secrets.toml
|
secrets.toml
|
||||||
secrets.yaml
|
secrets.yaml
|
||||||
|
secrets/
|
||||||
twitch_secrets.toml
|
twitch_secrets.toml
|
||||||
|
|
||||||
# Old env secrets
|
# Old env secrets
|
||||||
|
|||||||
@@ -0,0 +1,437 @@
|
|||||||
|
{
|
||||||
|
"title": "Storyteller-web — HTTP overview",
|
||||||
|
"description": "Per-request HTTP metrics emitted by the storyteller-web metrics middleware (`http.server.request.count` counter, `http.server.request.duration_ms` distribution). Filter by `env` / `service` / `route` using the template variables at the top.\n\nMetrics emitted from `crates/lib/metrics` + `crates/service/web/storyteller_web/src/http_server/middleware/metrics_middleware/`. Source of truth for this dashboard JSON is `_metrics/datadog/dashboards/storyteller_web_http_overview.json` in the storyteller monorepo. Apply changes with `script/metrics/apply_dashboards.sh`.",
|
||||||
|
"layout_type": "ordered",
|
||||||
|
"notify_list": [],
|
||||||
|
"template_variables": [
|
||||||
|
{
|
||||||
|
"name": "env",
|
||||||
|
"prefix": "env",
|
||||||
|
"available_values": [],
|
||||||
|
"default": "*"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "service",
|
||||||
|
"prefix": "service",
|
||||||
|
"available_values": [],
|
||||||
|
"default": "storyteller-web"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "route",
|
||||||
|
"prefix": "route",
|
||||||
|
"available_values": [],
|
||||||
|
"default": "*"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"reflow_type": "fixed",
|
||||||
|
"widgets": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "group",
|
||||||
|
"layout_type": "ordered",
|
||||||
|
"title": "Top-line health",
|
||||||
|
"show_title": true,
|
||||||
|
"widgets": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "query_value",
|
||||||
|
"title": "Request rate (req/s)",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"precision": 1,
|
||||||
|
"autoscale": true,
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "sum:http.server.request.count{$env,$service,$route}.as_rate()",
|
||||||
|
"aggregator": "avg"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"width": 3,
|
||||||
|
"height": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "query_value",
|
||||||
|
"title": "5xx error rate (%)",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"precision": 2,
|
||||||
|
"autoscale": false,
|
||||||
|
"custom_unit": "%",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "(sum:http.server.request.count{$env,$service,$route,status_class:5xx}.as_count() / sum:http.server.request.count{$env,$service,$route}.as_count()) * 100",
|
||||||
|
"aggregator": "avg",
|
||||||
|
"conditional_formats": [
|
||||||
|
{
|
||||||
|
"comparator": ">",
|
||||||
|
"value": 5.0,
|
||||||
|
"palette": "white_on_red"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comparator": ">",
|
||||||
|
"value": 1.0,
|
||||||
|
"palette": "white_on_yellow"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comparator": "<=",
|
||||||
|
"value": 1.0,
|
||||||
|
"palette": "white_on_green"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 3,
|
||||||
|
"y": 0,
|
||||||
|
"width": 3,
|
||||||
|
"height": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "query_value",
|
||||||
|
"title": "Latency p95 (ms)",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"precision": 0,
|
||||||
|
"autoscale": true,
|
||||||
|
"custom_unit": "ms",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "avg:http.server.request.duration_ms.p95{$env,$service,$route}",
|
||||||
|
"aggregator": "avg",
|
||||||
|
"conditional_formats": [
|
||||||
|
{
|
||||||
|
"comparator": ">",
|
||||||
|
"value": 1000,
|
||||||
|
"palette": "white_on_red"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comparator": ">",
|
||||||
|
"value": 250,
|
||||||
|
"palette": "white_on_yellow"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comparator": "<=",
|
||||||
|
"value": 250,
|
||||||
|
"palette": "white_on_green"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 6,
|
||||||
|
"y": 0,
|
||||||
|
"width": 3,
|
||||||
|
"height": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "query_value",
|
||||||
|
"title": "Latency p99 (ms)",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"precision": 0,
|
||||||
|
"autoscale": true,
|
||||||
|
"custom_unit": "ms",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "avg:http.server.request.duration_ms.p99{$env,$service,$route}",
|
||||||
|
"aggregator": "avg"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 9,
|
||||||
|
"y": 0,
|
||||||
|
"width": 3,
|
||||||
|
"height": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"width": 12,
|
||||||
|
"height": 3
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "group",
|
||||||
|
"layout_type": "ordered",
|
||||||
|
"title": "Traffic shape",
|
||||||
|
"show_title": true,
|
||||||
|
"widgets": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Requests/sec by status class (stacked)",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"show_legend": true,
|
||||||
|
"legend_layout": "horizontal",
|
||||||
|
"legend_columns": [
|
||||||
|
"avg",
|
||||||
|
"max",
|
||||||
|
"value",
|
||||||
|
"sum"
|
||||||
|
],
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "sum:http.server.request.count{$env,$service,$route} by {status_class}.as_rate()",
|
||||||
|
"display_type": "bars",
|
||||||
|
"style": {
|
||||||
|
"palette": "semantic",
|
||||||
|
"line_type": "solid",
|
||||||
|
"line_width": "normal"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"scale": "linear",
|
||||||
|
"include_zero": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"width": 6,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Latency percentiles over time",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"show_legend": true,
|
||||||
|
"legend_layout": "horizontal",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "avg:http.server.request.duration_ms.p50{$env,$service,$route}",
|
||||||
|
"display_type": "line",
|
||||||
|
"style": {
|
||||||
|
"palette": "cool",
|
||||||
|
"line_type": "solid",
|
||||||
|
"line_width": "normal"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"q": "avg:http.server.request.duration_ms.p95{$env,$service,$route}",
|
||||||
|
"display_type": "line",
|
||||||
|
"style": {
|
||||||
|
"palette": "warm",
|
||||||
|
"line_type": "solid",
|
||||||
|
"line_width": "normal"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"q": "avg:http.server.request.duration_ms.p99{$env,$service,$route}",
|
||||||
|
"display_type": "line",
|
||||||
|
"style": {
|
||||||
|
"palette": "warm",
|
||||||
|
"line_type": "dashed",
|
||||||
|
"line_width": "normal"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"scale": "linear",
|
||||||
|
"include_zero": true,
|
||||||
|
"label": "ms"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 6,
|
||||||
|
"y": 0,
|
||||||
|
"width": 6,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 3,
|
||||||
|
"width": 12,
|
||||||
|
"height": 5
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "group",
|
||||||
|
"layout_type": "ordered",
|
||||||
|
"title": "Per-route breakdown",
|
||||||
|
"show_title": true,
|
||||||
|
"widgets": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "toplist",
|
||||||
|
"title": "Top 10 routes by request volume",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "top(sum:http.server.request.count{$env,$service,$route} by {route}.as_count(), 10, 'sum', 'desc')"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"width": 6,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "toplist",
|
||||||
|
"title": "Top 10 routes by p95 latency",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "top(avg:http.server.request.duration_ms.p95{$env,$service,$route} by {route}, 10, 'mean', 'desc')"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 6,
|
||||||
|
"y": 0,
|
||||||
|
"width": 6,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "5xx error count by route",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"show_legend": true,
|
||||||
|
"legend_layout": "horizontal",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "sum:http.server.request.count{$env,$service,$route,status_class:5xx} by {route}.as_count()",
|
||||||
|
"display_type": "bars"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"scale": "linear",
|
||||||
|
"include_zero": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 4,
|
||||||
|
"width": 12,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "4xx error count by route",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"show_legend": true,
|
||||||
|
"legend_layout": "horizontal",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "sum:http.server.request.count{$env,$service,$route,status_class:4xx} by {route}.as_count()",
|
||||||
|
"display_type": "bars"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"scale": "linear",
|
||||||
|
"include_zero": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 8,
|
||||||
|
"width": 12,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 8,
|
||||||
|
"width": 12,
|
||||||
|
"height": 13
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "group",
|
||||||
|
"layout_type": "ordered",
|
||||||
|
"title": "Methods and status codes",
|
||||||
|
"show_title": true,
|
||||||
|
"widgets": [
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Requests/sec by HTTP method",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"show_legend": true,
|
||||||
|
"legend_layout": "horizontal",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "sum:http.server.request.count{$env,$service,$route} by {method}.as_rate()",
|
||||||
|
"display_type": "bars"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"width": 6,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"definition": {
|
||||||
|
"type": "toplist",
|
||||||
|
"title": "Status code distribution",
|
||||||
|
"title_size": "16",
|
||||||
|
"title_align": "left",
|
||||||
|
"requests": [
|
||||||
|
{
|
||||||
|
"q": "top(sum:http.server.request.count{$env,$service,$route} by {status_code}.as_count(), 25, 'sum', 'desc')"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 6,
|
||||||
|
"y": 0,
|
||||||
|
"width": 6,
|
||||||
|
"height": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 21,
|
||||||
|
"width": 12,
|
||||||
|
"height": 5
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": "thp-c27-vnv"
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"name": "[storyteller-web] HTTP 5xx error rate is elevated",
|
||||||
|
"type": "query alert",
|
||||||
|
"tags": [
|
||||||
|
"service:storyteller-web",
|
||||||
|
"team:backend",
|
||||||
|
"metric:http_5xx"
|
||||||
|
],
|
||||||
|
"priority": 2,
|
||||||
|
"message": "5xx error rate on storyteller-web is elevated.\n\nDashboard: https://app.datadoghq.com/dashboard/thp-c27-vnv/storyteller-web--http-overview\n\nMetric: `http.server.request.count{status_class:5xx} / http.server.request.count` over the last 5 minutes.\n\nFirst response steps:\n- Open the dashboard linked above. Check the **5xx error count by route** widget to identify which routes are failing.\n- Check the **Latency percentiles over time** widget — a latency spike often co-occurs with a 5xx spike (e.g. DB or upstream-provider stall).\n- Tail the production logs for `WARN`/`ERROR` lines from the affected route(s).\n\nThis monitor's source-of-truth JSON is at `_metrics/datadog/monitors/storyteller_web_5xx_error_rate.json`. Update with `script/metrics/apply_monitors.sh`.\n\n{{#is_alert}}🔥 5xx error rate is **{{value}}%**, above the **5%** critical threshold.{{/is_alert}}\n{{#is_warning}}⚠️ 5xx error rate is **{{value}}%**, above the **1%** warning threshold.{{/is_warning}}\n{{#is_recovery}}✅ 5xx error rate recovered to **{{value}}%**.{{/is_recovery}}\n\n@slack-storyteller-pages",
|
||||||
|
"query": "sum(last_5m):(sum:http.server.request.count{service:storyteller-web,status_class:5xx,env:production}.as_count() / sum:http.server.request.count{service:storyteller-web,env:production}.as_count()) * 100 > 5",
|
||||||
|
"options": {
|
||||||
|
"thresholds": {
|
||||||
|
"critical": 5,
|
||||||
|
"warning": 1
|
||||||
|
},
|
||||||
|
"notify_audit": false,
|
||||||
|
"require_full_window": false,
|
||||||
|
"notify_no_data": false,
|
||||||
|
"renotify_interval": 30,
|
||||||
|
"evaluation_delay": 60,
|
||||||
|
"new_host_delay": 60,
|
||||||
|
"include_tags": true,
|
||||||
|
"escalation_message": "5xx error rate on storyteller-web is **still elevated** — please ack or fix."
|
||||||
|
},
|
||||||
|
"id": 286427558
|
||||||
|
}
|
||||||
@@ -3,38 +3,43 @@
|
|||||||
use std::fs::read_to_string;
|
use std::fs::read_to_string;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// Reads the Datadog API key from `datadog_key.txt` at the workspace root.
|
/// Reads the Datadog API key from `secrets/datadog_key.txt` (relative to
|
||||||
/// Used only by `#[ignore]` smoke tests that hit the real API.
|
/// the workspace root). Used only by `#[ignore]`d smoke tests that hit the
|
||||||
|
/// real API.
|
||||||
pub(crate) fn get_test_api_key() -> Result<String, String> {
|
pub(crate) fn get_test_api_key() -> Result<String, String> {
|
||||||
let candidates = candidate_paths();
|
let candidates = candidate_paths();
|
||||||
for path in &candidates {
|
for path in &candidates {
|
||||||
if let Ok(contents) = read_to_string(path) {
|
if let Ok(contents) = read_to_string(path) {
|
||||||
let trimmed = contents.trim().to_string();
|
let trimmed = contents.trim().to_string();
|
||||||
if trimmed.is_empty() {
|
if trimmed.is_empty() {
|
||||||
return Err(format!("datadog_key.txt at {} is empty", path.display()));
|
return Err(format!("{} is empty", path.display()));
|
||||||
}
|
}
|
||||||
return Ok(trimmed);
|
return Ok(trimmed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(format!(
|
Err(format!(
|
||||||
"Could not locate datadog_key.txt. Tried: {}",
|
"Could not locate Datadog API key. Tried: {}",
|
||||||
candidates.iter().map(|p| p.display().to_string()).collect::<Vec<_>>().join(", "),
|
candidates.iter().map(|p| p.display().to_string()).collect::<Vec<_>>().join(", "),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Walk upward from the test's CWD looking for `datadog_key.txt`. Tests run
|
/// Walk upward from the test's CWD looking for the key. Tests run from
|
||||||
/// from the crate dir (`crates/api_clients/datadog_client`), so we need to
|
/// the crate dir (`crates/api_clients/datadog_client`), so we need to
|
||||||
/// climb a few levels to reach the workspace root.
|
/// climb a few levels to reach the workspace root. We try the new
|
||||||
|
/// `secrets/datadog_key.txt` location first and fall back to the older
|
||||||
|
/// root-level `datadog_key.txt` for any stale checkouts.
|
||||||
fn candidate_paths() -> Vec<PathBuf> {
|
fn candidate_paths() -> Vec<PathBuf> {
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
let mut current: PathBuf = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
|
let mut current: PathBuf = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
|
||||||
for _ in 0..6 {
|
for _ in 0..6 {
|
||||||
|
out.push(current.join("secrets/datadog_key.txt"));
|
||||||
out.push(current.join("datadog_key.txt"));
|
out.push(current.join("datadog_key.txt"));
|
||||||
if !current.pop() {
|
if !current.pop() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Last-resort hardcoded path matching this repo's known layout.
|
// Last-resort hardcoded paths matching this repo's known layout.
|
||||||
|
out.push(Path::new("/Users/bt/dev/storyteller/artcraft/secrets/datadog_key.txt").to_path_buf());
|
||||||
out.push(Path::new("/Users/bt/dev/storyteller/artcraft/datadog_key.txt").to_path_buf());
|
out.push(Path::new("/Users/bt/dev/storyteller/artcraft/datadog_key.txt").to_path_buf());
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|||||||
Executable
+168
@@ -0,0 +1,168 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# apply_dashboards.sh
|
||||||
|
#
|
||||||
|
# Idempotently applies every dashboard JSON under `_metrics/datadog/dashboards/`
|
||||||
|
# to our Datadog account.
|
||||||
|
#
|
||||||
|
# Behavior, per file:
|
||||||
|
# - If the file already has a top-level `"id": "<dd-dashboard-id>"`, PUT
|
||||||
|
# the dashboard to `/api/v1/dashboard/<id>` (in-place update).
|
||||||
|
# - Otherwise, POST to `/api/v1/dashboard` (create) and write the assigned
|
||||||
|
# id back into the local JSON so the next run becomes an update.
|
||||||
|
#
|
||||||
|
# Secrets:
|
||||||
|
# secrets/datadog_key.txt — DD-API-KEY (required)
|
||||||
|
# secrets/datadog_app_key.txt — DD-APPLICATION-KEY (required;
|
||||||
|
# dashboard endpoints reject API-key-only
|
||||||
|
# calls with 403).
|
||||||
|
#
|
||||||
|
# Both files should contain just the key, optionally with trailing whitespace.
|
||||||
|
#
|
||||||
|
# Flags:
|
||||||
|
# --dry-run Print what would happen; don't call the API.
|
||||||
|
# --site <host> Datadog site host. Default: api.datadoghq.com.
|
||||||
|
# US3 = api.us3.datadoghq.com, EU = api.datadoghq.eu, etc.
|
||||||
|
# --dir <path> Override dashboards directory.
|
||||||
|
# Default: _metrics/datadog/dashboards
|
||||||
|
#
|
||||||
|
# Examples:
|
||||||
|
# script/metrics/apply_dashboards.sh
|
||||||
|
# script/metrics/apply_dashboards.sh --dry-run
|
||||||
|
# script/metrics/apply_dashboards.sh --site api.datadoghq.eu
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# -------- defaults --------
|
||||||
|
DD_SITE="api.datadoghq.com"
|
||||||
|
DASH_DIR="_metrics/datadog/dashboards"
|
||||||
|
API_KEY_FILE="secrets/datadog_key.txt"
|
||||||
|
APP_KEY_FILE="secrets/datadog_app_key.txt"
|
||||||
|
DRY_RUN=0
|
||||||
|
|
||||||
|
# -------- args --------
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--dry-run) DRY_RUN=1; shift ;;
|
||||||
|
--site) DD_SITE="$2"; shift 2 ;;
|
||||||
|
--dir) DASH_DIR="$2"; shift 2 ;;
|
||||||
|
-h|--help)
|
||||||
|
sed -n '2,28p' "$0"; exit 0 ;;
|
||||||
|
*)
|
||||||
|
echo "unknown flag: $1" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
BASE="https://${DD_SITE}"
|
||||||
|
|
||||||
|
# -------- preflight --------
|
||||||
|
command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }
|
||||||
|
command -v curl >/dev/null 2>&1 || { echo "curl is required" >&2; exit 1; }
|
||||||
|
|
||||||
|
if [ ! -s "$API_KEY_FILE" ]; then
|
||||||
|
echo "missing or empty API key file: $API_KEY_FILE" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -s "$APP_KEY_FILE" ]; then
|
||||||
|
cat >&2 <<EOF
|
||||||
|
Missing or empty application key file: $APP_KEY_FILE
|
||||||
|
|
||||||
|
The dashboard API requires BOTH a Datadog API key AND an Application key.
|
||||||
|
Generate an Application key at:
|
||||||
|
https://app.datadoghq.com/organization-settings/application-keys
|
||||||
|
and save it as $APP_KEY_FILE (just the key, one line).
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
DD_API_KEY=$(tr -d '[:space:]' < "$API_KEY_FILE")
|
||||||
|
DD_APP_KEY=$(tr -d '[:space:]' < "$APP_KEY_FILE")
|
||||||
|
|
||||||
|
if [ ! -d "$DASH_DIR" ]; then
|
||||||
|
echo "no such directory: $DASH_DIR" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
shopt -s nullglob
|
||||||
|
files=("$DASH_DIR"/*.json)
|
||||||
|
if [ ${#files[@]} -eq 0 ]; then
|
||||||
|
echo "no dashboard JSON files in $DASH_DIR" >&2
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -------- per-file apply --------
|
||||||
|
exit_code=0
|
||||||
|
|
||||||
|
for file in "${files[@]}"; do
|
||||||
|
echo
|
||||||
|
echo "==> $file"
|
||||||
|
|
||||||
|
# Validate JSON before sending it anywhere.
|
||||||
|
if ! jq empty "$file" 2>/dev/null; then
|
||||||
|
echo " invalid JSON, skipping" >&2
|
||||||
|
exit_code=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
existing_id=$(jq -r '.id // empty' "$file")
|
||||||
|
|
||||||
|
if [ -n "$existing_id" ]; then
|
||||||
|
method="PUT"
|
||||||
|
url="${BASE}/api/v1/dashboard/${existing_id}"
|
||||||
|
echo " updating existing dashboard id=${existing_id}"
|
||||||
|
else
|
||||||
|
method="POST"
|
||||||
|
url="${BASE}/api/v1/dashboard"
|
||||||
|
echo " creating new dashboard"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
echo " [dry-run] would ${method} ${url}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
http_status_file=$(mktemp)
|
||||||
|
response_file=$(mktemp)
|
||||||
|
# Drop the local-only `id` field before sending — Datadog returns it on
|
||||||
|
# POST and accepts it (or ignores it) on PUT, but keeping it out of the
|
||||||
|
# request body avoids any chance of confusing the API.
|
||||||
|
payload=$(jq 'del(.id)' "$file")
|
||||||
|
|
||||||
|
curl_status=$(curl -sS -o "$response_file" -w "%{http_code}" \
|
||||||
|
-X "$method" "$url" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "DD-API-KEY: ${DD_API_KEY}" \
|
||||||
|
-H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \
|
||||||
|
--data-binary "$payload" || echo "000")
|
||||||
|
echo "$curl_status" > "$http_status_file"
|
||||||
|
|
||||||
|
if [ "$curl_status" -lt 200 ] || [ "$curl_status" -ge 300 ]; then
|
||||||
|
echo " ✗ HTTP $curl_status"
|
||||||
|
sed 's/^/ /' "$response_file" >&2
|
||||||
|
exit_code=1
|
||||||
|
else
|
||||||
|
if [ "$method" = "POST" ]; then
|
||||||
|
new_id=$(jq -r '.id // empty' "$response_file")
|
||||||
|
if [ -n "$new_id" ]; then
|
||||||
|
tmp=$(mktemp)
|
||||||
|
jq --arg id "$new_id" '. + {id: $id}' "$file" > "$tmp" && mv "$tmp" "$file"
|
||||||
|
echo " ✓ created (id=${new_id}), id written back to $file"
|
||||||
|
else
|
||||||
|
echo " ✓ created but response had no id (??); body:"
|
||||||
|
sed 's/^/ /' "$response_file"
|
||||||
|
exit_code=1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ✓ updated"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f "$http_status_file" "$response_file"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$exit_code" -eq 0 ]; then
|
||||||
|
echo "All dashboards applied."
|
||||||
|
else
|
||||||
|
echo "Finished with errors. See output above."
|
||||||
|
fi
|
||||||
|
exit "$exit_code"
|
||||||
Executable
+156
@@ -0,0 +1,156 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# apply_monitors.sh
|
||||||
|
#
|
||||||
|
# Idempotently applies every monitor JSON under `_metrics/datadog/monitors/`
|
||||||
|
# to our Datadog account. Sibling to `apply_dashboards.sh`.
|
||||||
|
#
|
||||||
|
# Behavior, per file:
|
||||||
|
# - If the file already has a top-level `"id": <numeric>`, PUT the
|
||||||
|
# monitor to `/api/v1/monitor/<id>` (in-place update).
|
||||||
|
# - Otherwise, POST to `/api/v1/monitor` (create) and write the assigned
|
||||||
|
# id back into the local JSON so the next run becomes an update.
|
||||||
|
#
|
||||||
|
# Note: Datadog dashboard ids are strings (`thp-c27-vnv`); monitor ids are
|
||||||
|
# integers. The script handles both — `jq` writes the value the API
|
||||||
|
# returns, whatever its type.
|
||||||
|
#
|
||||||
|
# Secrets:
|
||||||
|
# secrets/datadog_key.txt — DD-API-KEY (required)
|
||||||
|
# secrets/datadog_app_key.txt — DD-APPLICATION-KEY (required)
|
||||||
|
#
|
||||||
|
# Flags:
|
||||||
|
# --dry-run Print what would happen; don't call the API.
|
||||||
|
# --site <host> Datadog site host. Default: api.datadoghq.com.
|
||||||
|
# --dir <path> Override monitors directory.
|
||||||
|
# Default: _metrics/datadog/monitors
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# -------- defaults --------
|
||||||
|
DD_SITE="api.datadoghq.com"
|
||||||
|
MON_DIR="_metrics/datadog/monitors"
|
||||||
|
API_KEY_FILE="secrets/datadog_key.txt"
|
||||||
|
APP_KEY_FILE="secrets/datadog_app_key.txt"
|
||||||
|
DRY_RUN=0
|
||||||
|
|
||||||
|
# -------- args --------
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--dry-run) DRY_RUN=1; shift ;;
|
||||||
|
--site) DD_SITE="$2"; shift 2 ;;
|
||||||
|
--dir) MON_DIR="$2"; shift 2 ;;
|
||||||
|
-h|--help)
|
||||||
|
sed -n '2,24p' "$0"; exit 0 ;;
|
||||||
|
*)
|
||||||
|
echo "unknown flag: $1" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
BASE="https://${DD_SITE}"
|
||||||
|
|
||||||
|
# -------- preflight --------
|
||||||
|
command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }
|
||||||
|
command -v curl >/dev/null 2>&1 || { echo "curl is required" >&2; exit 1; }
|
||||||
|
|
||||||
|
if [ ! -s "$API_KEY_FILE" ]; then
|
||||||
|
echo "missing or empty API key file: $API_KEY_FILE" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -s "$APP_KEY_FILE" ]; then
|
||||||
|
cat >&2 <<EOF
|
||||||
|
Missing or empty application key file: $APP_KEY_FILE
|
||||||
|
|
||||||
|
The monitor API requires BOTH a Datadog API key AND an Application key.
|
||||||
|
Generate an Application key at:
|
||||||
|
https://app.datadoghq.com/organization-settings/application-keys
|
||||||
|
and save it as $APP_KEY_FILE (just the key, one line).
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
DD_API_KEY=$(tr -d '[:space:]' < "$API_KEY_FILE")
|
||||||
|
DD_APP_KEY=$(tr -d '[:space:]' < "$APP_KEY_FILE")
|
||||||
|
|
||||||
|
if [ ! -d "$MON_DIR" ]; then
|
||||||
|
echo "no such directory: $MON_DIR" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
shopt -s nullglob
|
||||||
|
files=("$MON_DIR"/*.json)
|
||||||
|
if [ ${#files[@]} -eq 0 ]; then
|
||||||
|
echo "no monitor JSON files in $MON_DIR" >&2
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit_code=0
|
||||||
|
|
||||||
|
for file in "${files[@]}"; do
|
||||||
|
echo
|
||||||
|
echo "==> $file"
|
||||||
|
|
||||||
|
if ! jq empty "$file" 2>/dev/null; then
|
||||||
|
echo " invalid JSON, skipping" >&2
|
||||||
|
exit_code=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
existing_id=$(jq -r '.id // empty' "$file")
|
||||||
|
|
||||||
|
if [ -n "$existing_id" ]; then
|
||||||
|
method="PUT"
|
||||||
|
url="${BASE}/api/v1/monitor/${existing_id}"
|
||||||
|
echo " updating existing monitor id=${existing_id}"
|
||||||
|
else
|
||||||
|
method="POST"
|
||||||
|
url="${BASE}/api/v1/monitor"
|
||||||
|
echo " creating new monitor"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
echo " [dry-run] would ${method} ${url}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
response_file=$(mktemp)
|
||||||
|
payload=$(jq 'del(.id)' "$file")
|
||||||
|
|
||||||
|
curl_status=$(curl -sS -o "$response_file" -w "%{http_code}" \
|
||||||
|
-X "$method" "$url" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "DD-API-KEY: ${DD_API_KEY}" \
|
||||||
|
-H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \
|
||||||
|
--data-binary "$payload" || echo "000")
|
||||||
|
|
||||||
|
if [ "$curl_status" -lt 200 ] || [ "$curl_status" -ge 300 ]; then
|
||||||
|
echo " ✗ HTTP $curl_status"
|
||||||
|
sed 's/^/ /' "$response_file" >&2
|
||||||
|
exit_code=1
|
||||||
|
else
|
||||||
|
if [ "$method" = "POST" ]; then
|
||||||
|
# Monitor ids are integers; preserve type when writing back.
|
||||||
|
new_id=$(jq -r '.id // empty' "$response_file")
|
||||||
|
if [ -n "$new_id" ]; then
|
||||||
|
tmp=$(mktemp)
|
||||||
|
jq --argjson id "$new_id" '. + {id: $id}' "$file" > "$tmp" && mv "$tmp" "$file"
|
||||||
|
echo " ✓ created (id=${new_id}), id written back to $file"
|
||||||
|
else
|
||||||
|
echo " ✓ created but response had no id (??); body:"
|
||||||
|
sed 's/^/ /' "$response_file"
|
||||||
|
exit_code=1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ✓ updated"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f "$response_file"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$exit_code" -eq 0 ]; then
|
||||||
|
echo "All monitors applied."
|
||||||
|
else
|
||||||
|
echo "Finished with errors. See output above."
|
||||||
|
fi
|
||||||
|
exit "$exit_code"
|
||||||
Reference in New Issue
Block a user