Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions api-server/src/incidents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ use std::collections::HashMap;
use std::sync::Mutex;

use axum::extract::Path;
use axum::http::StatusCode;
use axum::http::{header::AUTHORIZATION, HeaderMap, StatusCode};
use axum::Json;
use metrics::{counter, histogram};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use serde_json::json;
use sha2::{Digest, Sha256};

use crate::alerting::{self, Notification, Severity};

Expand Down Expand Up @@ -120,6 +121,7 @@ pub struct ActionItem {

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Postmortem {
#[serde(default)]
pub author: String,
pub summary: String,
pub impact: String,
Expand Down Expand Up @@ -524,6 +526,9 @@ impl IncidentManager {
if inc.status != IncidentStatus::Resolved {
return Err(IncidentError::InvalidState("postmortem can only be attached to a resolved incident"));
}
if inc.postmortem.is_some() {
return Err(IncidentError::InvalidState("postmortem has already been submitted"));
}
pm.submitted_at = now;
let author = pm.author.clone();
inc.postmortem = Some(pm);
Expand Down Expand Up @@ -711,7 +716,12 @@ pub async fn note_handler(Path(id): Path<String>, Json(req): Json<NoteRequest>)
}

/// `POST /v1/admin/incidents/{id}/postmortem`
pub async fn postmortem_handler(Path(id): Path<String>, Json(pm): Json<Postmortem>) -> ApiResult<Incident> {
pub async fn postmortem_handler(
Path(id): Path<String>,
crate::auth::AuthExtension(claims): crate::auth::AuthExtension,
Json(mut pm): Json<Postmortem>,
) -> ApiResult<Incident> {
pm.author = claims.sub;
INCIDENTS.submit_postmortem(&id, pm, alerting::now_secs()).map(Json).map_err(map_err)
}

Expand All @@ -721,9 +731,16 @@ pub struct AlertmanagerWebhook {
pub alerts: Vec<AlertmanagerAlert>,
}

#[derive(Debug, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum AlertmanagerStatus {
Firing,
Resolved,
}

#[derive(Debug, Deserialize)]
pub struct AlertmanagerAlert {
pub status: String,
pub status: AlertmanagerStatus,
#[serde(default)]
pub labels: HashMap<String, String>,
#[serde(default)]
Expand Down Expand Up @@ -762,8 +779,8 @@ pub async fn alertmanager_webhook_handler(
let (mut opened, mut resolved) = (Vec::new(), Vec::new());
for a in hook.alerts {
let name = a.labels.get("alertname").cloned().unwrap_or_else(|| "UnknownAlert".into());
let dedup_key = format!("am-{}", if a.fingerprint.is_empty() { &name } else { &a.fingerprint });
if a.status == "resolved" {
let dedup_key = format!("am-{}", a.fingerprint);
if a.status == AlertmanagerStatus::Resolved {
if let Some(inc) = INCIDENTS.resolve_by_key(&dedup_key, "alertmanager", now) {
resolved.push(inc.id);
}
Expand Down
51 changes: 27 additions & 24 deletions api-server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ async fn main() {
subscription_broadcaster.clone(),
);

let rate_limiter = Arc::new(rate_limit::RateLimitMiddleware::new(rate_limit::RateLimitConfig::default()));
let rate_limiter = rate_limit::RateLimitMiddleware::new(rate_limit::RateLimitConfig::default());

// Register every dependency in the container, then resolve the state
// through it. A missing dependency yields a typed error rather than a
Expand All @@ -419,6 +419,11 @@ async fn main() {
let request_queue = Arc::new(request_queue::RequestQueue::new(
request_queue::QueueConfig::default(),
));
let signed = middleware::from_fn(request_signing::verify_request_signature);
let signed_ip_routes = Router::new()
.route("/ip/commit", post(handlers::commit_ip))
.route("/ip/transfer", post(handlers::transfer_ip))
.route_layer(middleware::from_fn(request_signing::verify_request_signature));

// Mutating endpoints that move or commit IP/swap state require a signed
// request (see request_signing.rs) on top of the layers applied below.
Expand All @@ -443,7 +448,7 @@ async fn main() {
.route_layer(middleware::from_fn(request_signing::verify_request_signature));

let app = Router::new()
.merge(signed_routes)
.merge(signed_ip_routes)
.route("/health", get(health::health_handler))
.route("/health/detailed", get(health::detailed_health_handler))
.route("/metrics", get(metrics::metrics_handler))
Expand All @@ -457,15 +462,8 @@ async fn main() {
.route("/v1/admin/audit/suspicious-patterns", get(handlers::get_suspicious_patterns))
.route("/v1/admin/commitments/lifecycle", get(commitment_monitoring::lifecycle_handler))
.route("/v1/admin/alerts", get(commitment_monitoring::open_alerts_handler))
// #1068: incident management (PagerDuty / Opsgenie).
.route("/v1/admin/incidents", get(incidents::list_handler).post(incidents::create_handler))
.route("/v1/admin/incidents/stats", get(incidents::stats_handler))
.route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler))
.route("/v1/admin/incidents/{id}", get(incidents::get_handler))
.route("/v1/admin/incidents/{id}/acknowledge", post(incidents::acknowledge_handler))
.route("/v1/admin/incidents/{id}/resolve", post(incidents::resolve_handler))
.route("/v1/admin/incidents/{id}/notes", post(incidents::note_handler))
.route("/v1/admin/incidents/{id}/postmortem", post(incidents::postmortem_handler))
.merge(incident_routes())
.route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler))
.route("/v1/auth/recovery/initiate", post(account_recovery::initiate_recovery))
.route("/v1/auth/recovery/verify-token", post(account_recovery::verify_recovery_token))
.route("/v1/auth/recovery/questions", get(account_recovery::get_security_questions))
Expand Down Expand Up @@ -613,15 +611,8 @@ fn build_app() -> Router {
.route("/v1/admin/audit/suspicious-patterns", get(handlers::get_suspicious_patterns))
.route("/v1/admin/commitments/lifecycle", get(commitment_monitoring::lifecycle_handler))
.route("/v1/admin/alerts", get(commitment_monitoring::open_alerts_handler))
// #1068: incident management (PagerDuty / Opsgenie).
.route("/v1/admin/incidents", get(incidents::list_handler).post(incidents::create_handler))
.route("/v1/admin/incidents/stats", get(incidents::stats_handler))
.route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler))
.route("/v1/admin/incidents/{id}", get(incidents::get_handler))
.route("/v1/admin/incidents/{id}/acknowledge", post(incidents::acknowledge_handler))
.route("/v1/admin/incidents/{id}/resolve", post(incidents::resolve_handler))
.route("/v1/admin/incidents/{id}/notes", post(incidents::note_handler))
.route("/v1/admin/incidents/{id}/postmortem", post(incidents::postmortem_handler))
.merge(incident_routes())
.route("/v1/admin/incidents/alertmanager", post(incidents::alertmanager_webhook_handler))
.route("/v1/auth/recovery/initiate", post(account_recovery::initiate_recovery))
.route("/v1/auth/recovery/verify-token", post(account_recovery::verify_recovery_token))
.route("/v1/auth/recovery/questions", get(account_recovery::get_security_questions))
Expand Down Expand Up @@ -650,13 +641,13 @@ fn build_app() -> Router {
.route("/v1/ip/search/saved/{owner}", get(handlers::list_ip_saved_searches).post(handlers::save_ip_search))
.route("/v1/ip/search/saved/{owner}/{id}", axum::routing::delete(handlers::delete_ip_saved_search))
.route("/v1/ip/owner/{owner}/cursor", get(handlers::list_ip_by_owner_cursor))
.route("/v1/swap/initiate", post(handlers::initiate_swap))
.route("/v1/swap/batch-initiate", post(handlers::batch_initiate_swap))
.route("/v1/swap/bulk/initiate", post(handlers::batch_initiate_swap))
.route("/v1/swap/initiate", post(handlers::initiate_swap).layer(signed.clone()))
.route("/v1/swap/batch-initiate", post(handlers::batch_initiate_swap).layer(signed.clone()))
.route("/v1/swap/bulk/initiate", post(handlers::batch_initiate_swap).layer(signed.clone()))
.route("/v1/swap/{swap_id}/accept", post(handlers::accept_swap).layer(signed.clone()))
.route("/v1/swap/{swap_id}/reveal", post(handlers::reveal_key).layer(signed.clone()))
.route("/v1/swap/{swap_id}/cancel", post(handlers::cancel_swap).layer(signed.clone()))
.route("/v1/swap/{swap_id}/cancel-expired", post(handlers::cancel_expired_swap))
.route("/v1/swap/{swap_id}/cancel-expired", post(handlers::cancel_expired_swap).layer(signed.clone()))
.route("/v1/swap/{swap_id}", get(handlers::get_swap))
.route("/v1/swap/{swap_id}/escrow", get(handlers::get_swap_escrow))
.route("/v1/swap/{swap_id}/history", get(handlers::get_swap_history))
Expand Down Expand Up @@ -695,6 +686,18 @@ fn build_app() -> Router {
.layer(middleware::from_fn(require_json_content_type))
}

fn incident_routes() -> Router {
Router::new()
.route("/v1/admin/incidents", get(incidents::list_handler).post(incidents::create_handler))
.route("/v1/admin/incidents/stats", get(incidents::stats_handler))
.route("/v1/admin/incidents/{id}", get(incidents::get_handler))
.route("/v1/admin/incidents/{id}/acknowledge", post(incidents::acknowledge_handler))
.route("/v1/admin/incidents/{id}/resolve", post(incidents::resolve_handler))
.route("/v1/admin/incidents/{id}/notes", post(incidents::note_handler))
.route("/v1/admin/incidents/{id}/postmortem", post(incidents::postmortem_handler))
.route_layer(middleware::from_fn(auth::require_auth))
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
10 changes: 9 additions & 1 deletion docs/incident-response.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,15 @@ flowchart LR
evaluator, and Prometheus alerts via the Alertmanager `incident-tracker`
webhook (`POST /v1/admin/incidents/alertmanager`). Re-fires of the same alert
are deduplicated onto the open incident (`trigger_count` + timeline entry);
when the alert resolves, the incident resolves.
when the alert resolves, the incident resolves. Configure matching secret
values in the API server's `ALERTMANAGER_WEBHOOK_TOKEN` environment variable
and Alertmanager's mounted
`/etc/alertmanager/secrets/incident-tracker-token` file; the webhook rejects
requests if the API token is unset or does not match.
* **API access.** Incident administration endpoints require a valid access JWT
in `Authorization: Bearer …`. Postmortem authors are taken from the JWT
subject, not from the submitted body, and an incident's postmortem can only
be submitted once.
* **Paging.** Set `INCIDENT_PROVIDER=pagerduty` (+ `PAGERDUTY_ROUTING_KEY`) or
`INCIDENT_PROVIDER=opsgenie` (+ `OPSGENIE_API_KEY`, optional `OPSGENIE_API_URL`).
Incidents from in-process alerts and manual declarations are sent to the
Expand Down
2 changes: 2 additions & 0 deletions monitoring/alertmanager/alertmanager.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,5 @@ receivers:
- url: http://atomicip-api.atomicip-production.svc/v1/admin/incidents/alertmanager
send_resolved: true
max_alerts: 50
http_config:
bearer_token_file: /etc/alertmanager/secrets/incident-tracker-token
Loading