diff --git a/api-server/src/incidents.rs b/api-server/src/incidents.rs index 1d60a44..8c6a1bb 100644 --- a/api-server/src/incidents.rs +++ b/api-server/src/incidents.rs @@ -26,12 +26,13 @@ use std::collections::HashMap; use std::sync::Mutex; use axum::extract::Path; -use axum::http::StatusCode; +use axum::http::{header::AUTHORIZATION, HeaderMap, StatusCode}; use axum::Json; use metrics::{counter, histogram}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use serde_json::json; +use sha2::{Digest, Sha256}; use crate::alerting::{self, Notification, Severity}; @@ -120,6 +121,7 @@ pub struct ActionItem { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Postmortem { + #[serde(default)] pub author: String, pub summary: String, pub impact: String, @@ -524,6 +526,9 @@ impl IncidentManager { if inc.status != IncidentStatus::Resolved { return Err(IncidentError::InvalidState("postmortem can only be attached to a resolved incident")); } + if inc.postmortem.is_some() { + return Err(IncidentError::InvalidState("postmortem has already been submitted")); + } pm.submitted_at = now; let author = pm.author.clone(); inc.postmortem = Some(pm); @@ -711,7 +716,12 @@ pub async fn note_handler(Path(id): Path, Json(req): Json) } /// `POST /v1/admin/incidents/{id}/postmortem` -pub async fn postmortem_handler(Path(id): Path, Json(pm): Json) -> ApiResult { +pub async fn postmortem_handler( + Path(id): Path, + crate::auth::AuthExtension(claims): crate::auth::AuthExtension, + Json(mut pm): Json, +) -> ApiResult { + pm.author = claims.sub; INCIDENTS.submit_postmortem(&id, pm, alerting::now_secs()).map(Json).map_err(map_err) } @@ -721,9 +731,16 @@ pub struct AlertmanagerWebhook { pub alerts: Vec, } +#[derive(Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum AlertmanagerStatus { + Firing, + Resolved, +} + #[derive(Debug, Deserialize)] pub struct AlertmanagerAlert { - pub status: String, + pub status: AlertmanagerStatus, #[serde(default)] pub labels: HashMap, #[serde(default)] @@ -762,8 +779,8 @@ pub async fn alertmanager_webhook_handler( let (mut opened, mut resolved) = (Vec::new(), Vec::new()); for a in hook.alerts { let name = a.labels.get("alertname").cloned().unwrap_or_else(|| "UnknownAlert".into()); - let dedup_key = format!("am-{}", if a.fingerprint.is_empty() { &name } else { &a.fingerprint }); - if a.status == "resolved" { + let dedup_key = format!("am-{}", a.fingerprint); + if a.status == AlertmanagerStatus::Resolved { if let Some(inc) = INCIDENTS.resolve_by_key(&dedup_key, "alertmanager", now) { resolved.push(inc.id); } diff --git a/api-server/src/main.rs b/api-server/src/main.rs index 2be3734..f346835 100644 --- a/api-server/src/main.rs +++ b/api-server/src/main.rs @@ -394,7 +394,7 @@ async fn main() { subscription_broadcaster.clone(), ); - let rate_limiter = Arc::new(rate_limit::RateLimitMiddleware::new(rate_limit::RateLimitConfig::default())); + let rate_limiter = rate_limit::RateLimitMiddleware::new(rate_limit::RateLimitConfig::default()); // Register every dependency in the container, then resolve the state // through it. A missing dependency yields a typed error rather than a @@ -419,6 +419,11 @@ async fn main() { let request_queue = Arc::new(request_queue::RequestQueue::new( request_queue::QueueConfig::default(), )); + let signed = middleware::from_fn(request_signing::verify_request_signature); + let signed_ip_routes = Router::new() + .route("/ip/commit", post(handlers::commit_ip)) + .route("/ip/transfer", post(handlers::transfer_ip)) + .route_layer(middleware::from_fn(request_signing::verify_request_signature)); // Mutating endpoints that move or commit IP/swap state require a signed // request (see request_signing.rs) on top of the layers applied below. @@ -443,7 +448,7 @@ async fn main() { .route_layer(middleware::from_fn(request_signing::verify_request_signature)); let app = Router::new() - .merge(signed_routes) + .merge(signed_ip_routes) .route("/health", get(health::health_handler)) .route("/health/detailed", get(health::detailed_health_handler)) .route("/metrics", get(metrics::metrics_handler)) @@ -457,15 +462,8 @@ async fn main() { .route("/v1/admin/audit/suspicious-patterns", get(handlers::get_suspicious_patterns)) .route("/v1/admin/commitments/lifecycle", get(commitment_monitoring::lifecycle_handler)) .route("/v1/admin/alerts", get(commitment_monitoring::open_alerts_handler)) - // #1068: incident management (PagerDuty / Opsgenie). - .route("/v1/admin/incidents", get(incidents::list_handler).post(incidents::create_handler)) - .route("/v1/admin/incidents/stats", get(incidents::stats_handler)) - .route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler)) - .route("/v1/admin/incidents/{id}", get(incidents::get_handler)) - .route("/v1/admin/incidents/{id}/acknowledge", post(incidents::acknowledge_handler)) - .route("/v1/admin/incidents/{id}/resolve", post(incidents::resolve_handler)) - .route("/v1/admin/incidents/{id}/notes", post(incidents::note_handler)) - .route("/v1/admin/incidents/{id}/postmortem", post(incidents::postmortem_handler)) + .merge(incident_routes()) + .route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler)) .route("/v1/auth/recovery/initiate", post(account_recovery::initiate_recovery)) .route("/v1/auth/recovery/verify-token", post(account_recovery::verify_recovery_token)) .route("/v1/auth/recovery/questions", get(account_recovery::get_security_questions)) @@ -613,15 +611,8 @@ fn build_app() -> Router { .route("/v1/admin/audit/suspicious-patterns", get(handlers::get_suspicious_patterns)) .route("/v1/admin/commitments/lifecycle", get(commitment_monitoring::lifecycle_handler)) .route("/v1/admin/alerts", get(commitment_monitoring::open_alerts_handler)) - // #1068: incident management (PagerDuty / Opsgenie). - .route("/v1/admin/incidents", get(incidents::list_handler).post(incidents::create_handler)) - .route("/v1/admin/incidents/stats", get(incidents::stats_handler)) - .route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler)) - .route("/v1/admin/incidents/{id}", get(incidents::get_handler)) - .route("/v1/admin/incidents/{id}/acknowledge", post(incidents::acknowledge_handler)) - .route("/v1/admin/incidents/{id}/resolve", post(incidents::resolve_handler)) - .route("/v1/admin/incidents/{id}/notes", post(incidents::note_handler)) - .route("/v1/admin/incidents/{id}/postmortem", post(incidents::postmortem_handler)) + .merge(incident_routes()) + .route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler)) .route("/v1/auth/recovery/initiate", post(account_recovery::initiate_recovery)) .route("/v1/auth/recovery/verify-token", post(account_recovery::verify_recovery_token)) .route("/v1/auth/recovery/questions", get(account_recovery::get_security_questions)) @@ -650,13 +641,13 @@ fn build_app() -> Router { .route("/v1/ip/search/saved/{owner}", get(handlers::list_ip_saved_searches).post(handlers::save_ip_search)) .route("/v1/ip/search/saved/{owner}/{id}", axum::routing::delete(handlers::delete_ip_saved_search)) .route("/v1/ip/owner/{owner}/cursor", get(handlers::list_ip_by_owner_cursor)) - .route("/v1/swap/initiate", post(handlers::initiate_swap)) - .route("/v1/swap/batch-initiate", post(handlers::batch_initiate_swap)) - .route("/v1/swap/bulk/initiate", post(handlers::batch_initiate_swap)) + .route("/v1/swap/initiate", post(handlers::initiate_swap).layer(signed.clone())) + .route("/v1/swap/batch-initiate", post(handlers::batch_initiate_swap).layer(signed.clone())) + .route("/v1/swap/bulk/initiate", post(handlers::batch_initiate_swap).layer(signed.clone())) .route("/v1/swap/{swap_id}/accept", post(handlers::accept_swap).layer(signed.clone())) .route("/v1/swap/{swap_id}/reveal", post(handlers::reveal_key).layer(signed.clone())) .route("/v1/swap/{swap_id}/cancel", post(handlers::cancel_swap).layer(signed.clone())) - .route("/v1/swap/{swap_id}/cancel-expired", post(handlers::cancel_expired_swap)) + .route("/v1/swap/{swap_id}/cancel-expired", post(handlers::cancel_expired_swap).layer(signed.clone())) .route("/v1/swap/{swap_id}", get(handlers::get_swap)) .route("/v1/swap/{swap_id}/escrow", get(handlers::get_swap_escrow)) .route("/v1/swap/{swap_id}/history", get(handlers::get_swap_history)) @@ -695,6 +686,18 @@ fn build_app() -> Router { .layer(middleware::from_fn(require_json_content_type)) } +fn incident_routes() -> Router { + Router::new() + .route("/v1/admin/incidents", get(incidents::list_handler).post(incidents::create_handler)) + .route("/v1/admin/incidents/stats", get(incidents::stats_handler)) + .route("/v1/admin/incidents/{id}", get(incidents::get_handler)) + .route("/v1/admin/incidents/{id}/acknowledge", post(incidents::acknowledge_handler)) + .route("/v1/admin/incidents/{id}/resolve", post(incidents::resolve_handler)) + .route("/v1/admin/incidents/{id}/notes", post(incidents::note_handler)) + .route("/v1/admin/incidents/{id}/postmortem", post(incidents::postmortem_handler)) + .route_layer(middleware::from_fn(auth::require_auth)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/docs/incident-response.md b/docs/incident-response.md index c4c55ae..727aee3 100644 --- a/docs/incident-response.md +++ b/docs/incident-response.md @@ -21,7 +21,15 @@ flowchart LR evaluator, and Prometheus alerts via the Alertmanager `incident-tracker` webhook (`POST /v1/admin/incidents/alertmanager`). Re-fires of the same alert are deduplicated onto the open incident (`trigger_count` + timeline entry); - when the alert resolves, the incident resolves. + when the alert resolves, the incident resolves. Configure matching secret + values in the API server's `ALERTMANAGER_WEBHOOK_TOKEN` environment variable + and Alertmanager's mounted + `/etc/alertmanager/secrets/incident-tracker-token` file; the webhook rejects + requests if the API token is unset or does not match. +* **API access.** Incident administration endpoints require a valid access JWT + in `Authorization: Bearer …`. Postmortem authors are taken from the JWT + subject, not from the submitted body, and an incident's postmortem can only + be submitted once. * **Paging.** Set `INCIDENT_PROVIDER=pagerduty` (+ `PAGERDUTY_ROUTING_KEY`) or `INCIDENT_PROVIDER=opsgenie` (+ `OPSGENIE_API_KEY`, optional `OPSGENIE_API_URL`). Incidents from in-process alerts and manual declarations are sent to the diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml index 72f31c0..ece4e4b 100644 --- a/monitoring/alertmanager/alertmanager.yml +++ b/monitoring/alertmanager/alertmanager.yml @@ -98,3 +98,5 @@ receivers: - url: http://atomicip-api.atomicip-production.svc/v1/admin/incidents/alertmanager send_resolved: true max_alerts: 50 + http_config: + bearer_token_file: /etc/alertmanager/secrets/incident-tracker-token