From c7e52f1c3a907005bca4560b1d2430194dd7b54f Mon Sep 17 00:00:00 2001 From: ernescz Date: Sat, 5 Sep 2026 23:18:19 +0300 Subject: [PATCH 01/31] feat(situation): define history and delivery contracts Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/config/config.go | 27 +- internal/config/config_situations_test.go | 77 +++ internal/situation/model/history.go | 774 ++++++++++++++++++++++ internal/situation/model/history_test.go | 722 ++++++++++++++++++++ internal/situation/model/model.go | 19 + internal/situation/model/model_test.go | 36 + 6 files changed, 1652 insertions(+), 3 deletions(-) create mode 100644 internal/situation/model/history.go create mode 100644 internal/situation/model/history_test.go create mode 100644 internal/situation/model/model_test.go diff --git a/internal/config/config.go b/internal/config/config.go index 3c9400d..f92a18d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -106,9 +106,10 @@ const ( // concurrency, the reconcile poll interval, Situation claim lease/heartbeat // timing, the webhook source recovery grace, internal cadence tiers, the // fixed L2 call/work-attempt ceiling, the per-attempt wall clock, the shared -// L2 provider semaphore, and bounded retry/jitter. It carries no Plan 3/4 -// settings: no L1 call budget, connector concurrency, or envelope review -// interval. In particular, Plan 2 deliberately never adds +// L2 provider semaphore, and bounded retry/jitter — plus Plan 3's single +// Slack policy setting (Slack, SituationSlackConfig). It carries no other +// Plan 3/4 settings: no L1 call budget, connector concurrency, or envelope +// review interval. In particular, Plan 2 deliberately never adds // situations.budgets.max_l1_llm_calls (spec.md 02-controller-triage-coordination // "Attempt identity and completion": Acute Triage keeps its shipped // five-attempt schedule; a parsed budget with no distinct consuming behavior @@ -126,6 +127,22 @@ type SituationsConfig struct { AttemptWallSeconds int `yaml:"attempt_wall_seconds"` LLMConcurrency int `yaml:"llm_concurrency"` Retry SituationsRetryConfig `yaml:"retry"` + Slack SituationSlackConfig `yaml:"slack"` +} + +// SituationSlackConfig is Plan 3's only Situation-Slack policy setting: how +// long a warranted required-action change must wait after a delivered +// main-channel poke before it may create another one (the repage cooldown +// in "a materially changed required action after the configured cooldown", +// spec.md "Publication authority and Interruption priority"). Every other +// Slack delivery behavior — retry timing, lease/heartbeat, batch size, +// attempt accounting — reuses Plan 2's existing situations.retry/lease/ +// heartbeat settings and NotificationWorkerConfig's fixed spec constants; +// Plan 3 adds no duplicate notification knobs, no attempt ceiling (delivery +// retries indefinitely), and no Slack channel-history or +// read-before-redrive reconciliation setting. +type SituationSlackConfig struct { + RepageCooldownSeconds int `yaml:"repage_cooldown_seconds"` } // SituationsCadenceConfig sizes the controller's internal fast/normal/slow @@ -661,6 +678,9 @@ func Defaults() Config { MaxSeconds: 300, JitterPercent: 20, }, + Slack: SituationSlackConfig{ + RepageCooldownSeconds: 900, + }, }, Telemetry: TelemetryConfig{ OTLP: OTLPConfig{ @@ -1089,6 +1109,7 @@ func (c *Config) validateSituations() []string { {"situations.llm_concurrency", s.LLMConcurrency}, {"situations.retry.min_seconds", s.Retry.MinSeconds}, {"situations.retry.max_seconds", s.Retry.MaxSeconds}, + {"situations.slack.repage_cooldown_seconds", s.Slack.RepageCooldownSeconds}, } for _, p := range positive { if p.v <= 0 { diff --git a/internal/config/config_situations_test.go b/internal/config/config_situations_test.go index 4524e42..0dd2837 100644 --- a/internal/config/config_situations_test.go +++ b/internal/config/config_situations_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/alertint/alertint-agent/internal/situation/model" ) // situationsBaseYAML is a minimal valid config with the SQLite path @@ -45,6 +47,7 @@ func TestSituationsDefaults(t *testing.T) { {"retry.min_seconds", s.Retry.MinSeconds, 5}, {"retry.max_seconds", s.Retry.MaxSeconds, 300}, {"retry.jitter_percent", s.Retry.JitterPercent, 20}, + {"slack.repage_cooldown_seconds", s.Slack.RepageCooldownSeconds, 900}, } for _, c := range checks { if c.got != c.want { @@ -110,6 +113,8 @@ func TestSituationsValidation(t *testing.T) { {"max_l2_calls_per_attempt above fixed value", func(s *SituationsConfig) { s.MaxL2CallsPerAttempt = 3 }, true}, {"max_work_attempts_per_input below fixed value", func(s *SituationsConfig) { s.MaxWorkAttemptsPerInput = 4 }, true}, {"max_work_attempts_per_input above fixed value", func(s *SituationsConfig) { s.MaxWorkAttemptsPerInput = 6 }, true}, + {"slack.repage_cooldown_seconds zero", func(s *SituationsConfig) { s.Slack.RepageCooldownSeconds = 0 }, true}, + {"slack.repage_cooldown_seconds negative", func(s *SituationsConfig) { s.Slack.RepageCooldownSeconds = -1 }, true}, } for _, tc := range cases { @@ -192,3 +197,75 @@ situations: t.Errorf("default max_l2_calls_per_attempt = %d, want 2", cfg.Situations.MaxL2CallsPerAttempt) } } + +// TestLoad_SituationsSlackValidAndDefaults proves situations.slack. +// repage_cooldown_seconds loads and overrides the 900-second default. +func TestLoad_SituationsSlackValidAndDefaults(t *testing.T) { + yaml := situationsBaseYAML(t) + ` +situations: + slack: + repage_cooldown_seconds: 600 +` + path := writeConfig(t, yaml) + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Situations.Slack.RepageCooldownSeconds != 600 { + t.Errorf("slack.repage_cooldown_seconds = %d, want 600", cfg.Situations.Slack.RepageCooldownSeconds) + } +} + +// TestLoad_SituationsSlackRejectsAttemptCeiling proves Plan 3 adds no +// notification-worker attempt-ceiling knob under situations.slack: +// NotificationWorkerConfig retries valid Slack effects indefinitely (plan.md +// Cross-Task Contracts, "It has no maximum attempts field"), so a configured +// ceiling must fail strict decoding rather than being silently accepted and +// ignored. +func TestLoad_SituationsSlackRejectsAttemptCeiling(t *testing.T) { + yaml := situationsBaseYAML(t) + ` +situations: + slack: + max_attempts: 5 +` + path := writeConfig(t, yaml) + if _, err := Load(path); err == nil { + t.Fatal("expected strict-decode error for situations.slack.max_attempts") + } +} + +// TestLoad_SituationsSlackRejectsReadReconciliation proves Plan 3 adds no +// Slack channel-history-read or read-before-redrive reconciliation setting +// (Global Constraints: "Do not request Slack channel-history scopes and do +// not implement read-before-redrive reconciliation"). +func TestLoad_SituationsSlackRejectsReadReconciliation(t *testing.T) { + yaml := situationsBaseYAML(t) + ` +situations: + slack: + read_before_redrive: true +` + path := writeConfig(t, yaml) + if _, err := Load(path); err == nil { + t.Fatal("expected strict-decode error for situations.slack.read_before_redrive") + } +} + +// TestNotifySlackMinSeverityIsInterruptionPriorityFloor documents the +// compatibility contract (spec.md "Publication authority and Interruption +// priority"): notify.slack.min_severity keeps its existing accepted values +// low|medium|high unchanged, but in the Situation path the selected value is +// read only as a minimum deterministic Interruption priority — never Alert +// or model severity. Every accepted min_severity value must therefore +// already be one of model.InterruptionPriority's closed values; "critical" +// is a valid Interruption priority with no min_severity equivalent, so the +// compatibility setting can never select it as a floor. +func TestNotifySlackMinSeverityIsInterruptionPriorityFloor(t *testing.T) { + for _, v := range []string{"low", "medium", "high"} { + if err := model.InterruptionPriority(v).Validate(); err != nil { + t.Errorf("notify.slack.min_severity value %q must be a valid Interruption priority: %v", v, err) + } + } + if err := model.InterruptionPriority("critical").Validate(); err != nil { + t.Errorf("critical must be a valid Interruption priority: %v", err) + } +} diff --git a/internal/situation/model/history.go b/internal/situation/model/history.go new file mode 100644 index 0000000..c32c206 --- /dev/null +++ b/internal/situation/model/history.go @@ -0,0 +1,774 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package model + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// Bounded lengths for the free-text and array fields this file defines. +// These mirror the sizing already used elsewhere for similar fields +// (internal/situation's maxBoundedTextLength = 2000) rather than inventing +// new conventions: a short rendered line, a longer bounded detail/summary, +// a generous identifier/coordinate bound, and a bounded reference count. +const ( + // maxJournalHeadlineLength bounds JournalData.Headline — a short, + // rendered-facing summary line, never free-form prose. + maxJournalHeadlineLength = 200 + // maxJournalDetailLength bounds JournalData.Detail and every other + // bounded free-text field a Transition or NotificationIntent carries + // (e.g. AssessmentConclusion.SufficientReasonSummary). + maxJournalDetailLength = 2000 + // maxIdentifierLength bounds every identifier/coordinate-shaped string + // field (IDs, idempotency/client-message keys, Slack channel/message + // coordinates, error classes) — generous for any UUID or Slack ID shape + // without being unbounded. + maxIdentifierLength = 200 + // maxEvidenceRefs bounds how many evidence references a single + // Transition may cite. + maxEvidenceRefs = 50 +) + +// requireUTC reports an error unless t carries the UTC location. Every +// timestamp this package persists is normalized to UTC before storage +// (internal/store consistently calls time.Now().UTC()), so a non-UTC value +// signals the value never round-tripped through the store. field names just +// the JSON field (e.g. "created_at") — callers add their own ": " scope +// when wrapping, matching the rest of this package's convention of a bare +// type-name prefix (e.g. "action_contract: ...") on hand-rolled errors. +func requireUTC(field string, t time.Time) error { + if t.Location() != time.UTC { + return fmt.Errorf("%s: must be UTC, got location %s", field, t.Location()) + } + return nil +} + +// requireNonZeroUTC reports an error unless t is both set and UTC. +func requireNonZeroUTC(field string, t time.Time) error { + if t.IsZero() { + return fmt.Errorf("%s: is required", field) + } + return requireUTC(field, t) +} + +// ---------------------------------------------------------------------- +// Interruption priority +// ---------------------------------------------------------------------- + +// InterruptionPriority is the controller-derived, deterministic priority +// that governs whether a Transition may create a new main-channel poke. It +// is never Alert severity or model-authored severity (spec.md "Publication +// authority and Interruption priority"). +type InterruptionPriority string + +const ( + InterruptionLow InterruptionPriority = "low" + InterruptionMedium InterruptionPriority = "medium" + InterruptionHigh InterruptionPriority = "high" + InterruptionCritical InterruptionPriority = "critical" +) + +// Validate reports an error unless p is one of the closed InterruptionPriority +// values. +func (p InterruptionPriority) Validate() error { + return validateEnum("interruption_priority", p, + InterruptionLow, InterruptionMedium, InterruptionHigh, InterruptionCritical) +} + +// rank returns p's position in the deterministic low < medium < high < +// critical ranking, or -1 for an unknown value. +func (p InterruptionPriority) rank() int { + switch p { + case InterruptionLow: + return 0 + case InterruptionMedium: + return 1 + case InterruptionHigh: + return 2 + case InterruptionCritical: + return 3 + default: + return -1 + } +} + +// Less reports whether p ranks strictly below other in the deterministic +// low < medium < high < critical ranking. +func (p InterruptionPriority) Less(other InterruptionPriority) bool { + return p.rank() < other.rank() +} + +// ---------------------------------------------------------------------- +// Transition reason, actor, journal kind +// ---------------------------------------------------------------------- + +// TransitionReason is the closed reason recorded on one immutable +// Transition: exactly which material-change catalog entry produced it. +type TransitionReason string + +const ( + ReasonFirstAuthoritativeState TransitionReason = "first_authoritative_state" + ReasonMaterialAssessmentChanged TransitionReason = "material_assessment_changed" + ReasonAttentionChanged TransitionReason = "attention_changed" + ReasonOperatorContractChanged TransitionReason = "operator_contract_changed" + ReasonInvestigationStarted TransitionReason = "investigation_started" + ReasonInvestigationConcluded TransitionReason = "investigation_concluded" + ReasonRecoveryObserved TransitionReason = "recovery_observed" + ReasonRecoveryFailed TransitionReason = "recovery_failed" + ReasonRecovered TransitionReason = "recovered" + ReasonClosedUnknown TransitionReason = "closed_unknown" + ReasonRecurrenceMilestone TransitionReason = "recurrence_milestone" + ReasonTriageStateChanged TransitionReason = "triage_state_changed" + ReasonOperatorArtifactRecorded TransitionReason = "operator_artifact_recorded" +) + +// Validate reports an error unless r is one of the closed TransitionReason +// values. +func (r TransitionReason) Validate() error { + return validateEnum("transition_reason", r, + ReasonFirstAuthoritativeState, ReasonMaterialAssessmentChanged, ReasonAttentionChanged, + ReasonOperatorContractChanged, ReasonInvestigationStarted, ReasonInvestigationConcluded, + ReasonRecoveryObserved, ReasonRecoveryFailed, ReasonRecovered, ReasonClosedUnknown, + ReasonRecurrenceMilestone, ReasonTriageStateChanged, ReasonOperatorArtifactRecorded) +} + +// TransitionActor is the closed authority that produced one Transition. +type TransitionActor string + +const ( + ActorDeterministicController TransitionActor = "deterministic_controller" + ActorLLM TransitionActor = "llm" + ActorAttributedOperator TransitionActor = "attributed_operator" + // ActorOperatorPolicy is a defined value reserved for Plan 5's operator + // policy authority. Plan 3 cannot produce it — Validate rejects it + // unconditionally (spec.md "Domain model": "Plan 3 cannot produce + // operator_policy; the value is reserved for Plan 5 and rejected unless + // an attributed policy artifact exists"). + ActorOperatorPolicy TransitionActor = "operator_policy" +) + +// Validate reports an error unless a is one of the three Transition actors +// Plan 3 may produce: deterministic_controller, llm, or attributed_operator. +// ActorOperatorPolicy is a defined constant for a later plan and is always +// rejected here. +func (a TransitionActor) Validate() error { + return validateEnum("transition_actor", a, + ActorDeterministicController, ActorLLM, ActorAttributedOperator) +} + +// JournalKind is the closed shape of the immutable journal-render data a +// Transition carries, or JournalNone when the Transition creates no +// journal entry. +type JournalKind string + +const ( + JournalNone JournalKind = "none" + JournalPublication JournalKind = "publication" + JournalInvestigationStarted JournalKind = "investigation_started" + JournalInvestigationChanged JournalKind = "investigation_changed" + JournalEvidenceConclusion JournalKind = "evidence_conclusion" + JournalOperatorContractChanged JournalKind = "operator_contract_changed" + JournalRecoveryPending JournalKind = "recovery_pending" + JournalRecoveryRefired JournalKind = "recovery_refired" + JournalRecurrenceMilestone JournalKind = "recurrence_milestone" + JournalRecovered JournalKind = "recovered" + JournalClosedUnknown JournalKind = "closed_unknown" + JournalOperatorNote JournalKind = "operator_note" + JournalCapturedVerdict JournalKind = "captured_verdict" +) + +// Validate reports an error unless k is one of the closed JournalKind +// values. +func (k JournalKind) Validate() error { + return validateEnum("journal_kind", k, + JournalNone, JournalPublication, JournalInvestigationStarted, JournalInvestigationChanged, + JournalEvidenceConclusion, JournalOperatorContractChanged, JournalRecoveryPending, + JournalRecoveryRefired, JournalRecurrenceMilestone, JournalRecovered, JournalClosedUnknown, + JournalOperatorNote, JournalCapturedVerdict) +} + +// JournalData is the bounded, immutable render payload for one Transition's +// journal entry. It never carries unbounded operator-authored prose. +type JournalData struct { + Headline string `json:"headline"` + Detail string `json:"detail,omitempty"` + AttributedActor string `json:"attributed_actor,omitempty"` + ActionStatus string `json:"action_status,omitempty"` + RecurrenceCount int `json:"recurrence_count,omitempty"` + Delayed bool `json:"delayed,omitempty"` + NoLongerCurrent bool `json:"no_longer_current,omitempty"` + OccurredAt time.Time `json:"occurred_at"` +} + +// Validate checks JournalData's bounded lengths and its required, UTC +// occurred_at instant. +func (j JournalData) Validate() error { + if len(j.Headline) > maxJournalHeadlineLength { + return fmt.Errorf("journal_data: headline exceeds %d bytes", maxJournalHeadlineLength) + } + if len(j.Detail) > maxJournalDetailLength { + return fmt.Errorf("journal_data: detail exceeds %d bytes", maxJournalDetailLength) + } + if len(j.AttributedActor) > maxIdentifierLength { + return fmt.Errorf("journal_data: attributed_actor exceeds %d bytes", maxIdentifierLength) + } + if len(j.ActionStatus) > maxIdentifierLength { + return fmt.Errorf("journal_data: action_status exceeds %d bytes", maxIdentifierLength) + } + if j.RecurrenceCount < 0 { + return errors.New("journal_data: recurrence_count must be >= 0") + } + if err := requireNonZeroUTC("occurred_at", j.OccurredAt); err != nil { + return fmt.Errorf("journal_data: %w", err) + } + return nil +} + +// ---------------------------------------------------------------------- +// Assessment conclusion and projection facts (R3) +// ---------------------------------------------------------------------- + +// AssessmentConclusion is the closed, bounded slice of an authoritative +// Assessment's conclusion a Transition's ProjectionFacts may carry: closed +// judgment codes plus the bounded Sufficient-reason summary, never the full +// Assessment. +type AssessmentConclusion struct { + Persistence Persistence `json:"persistence"` + Impact Impact `json:"impact"` + Novelty Novelty `json:"novelty"` + Causality Causality `json:"causality"` + EvidenceQuality EvidenceQuality `json:"evidence_quality"` + LimitationCodes []string `json:"limitation_codes"` + SufficientReasonCode string `json:"sufficient_reason_code,omitempty"` + SufficientReasonSummary string `json:"sufficient_reason_summary,omitempty"` +} + +// MarshalJSON canonicalizes LimitationCodes to [] before marshaling: a +// nil-constructed AssessmentConclusion must never serialize +// limitation_codes as JSON null. +func (a AssessmentConclusion) MarshalJSON() ([]byte, error) { + type assessmentConclusionAlias AssessmentConclusion + out := assessmentConclusionAlias(a) + out.LimitationCodes = canonicalizeSlice(out.LimitationCodes) + return json.Marshal(out) +} + +// Validate checks AssessmentConclusion's closed judgment codes and the +// bounded Sufficient-reason summary (ProjectionFacts carries "no prose +// beyond the bounded Sufficient-reason summary"). +func (a AssessmentConclusion) Validate() error { + if err := a.Persistence.Validate(); err != nil { + return fmt.Errorf("assessment_conclusion: %w", err) + } + if err := a.Impact.Validate(); err != nil { + return fmt.Errorf("assessment_conclusion: %w", err) + } + if err := a.Novelty.Validate(); err != nil { + return fmt.Errorf("assessment_conclusion: %w", err) + } + if err := a.Causality.Validate(); err != nil { + return fmt.Errorf("assessment_conclusion: %w", err) + } + if err := a.EvidenceQuality.Validate(); err != nil { + return fmt.Errorf("assessment_conclusion: %w", err) + } + if len(a.SufficientReasonSummary) > maxJournalDetailLength { + return fmt.Errorf("assessment_conclusion: sufficient_reason_summary exceeds %d bytes", maxJournalDetailLength) + } + return nil +} + +// ProjectionFacts is the bounded, immutable slice of the coherent claim the +// Episode fold and renderers may read (R3). Closed codes and instants only; +// no prose beyond the bounded Sufficient-reason summary. +type ProjectionFacts struct { + PublicHandle *string `json:"public_handle,omitempty"` + EffectiveStartedAt time.Time `json:"effective_started_at"` + EffectiveStartedAtBasis SourceTimeBasis `json:"effective_started_at_basis"` + RecoveryObservedAt *time.Time `json:"recovery_observed_at,omitempty"` + GraceUntil *time.Time `json:"grace_until,omitempty"` + TerminalAt *time.Time `json:"terminal_at,omitempty"` + TerminalReason *TerminalReason `json:"terminal_reason,omitempty"` + Assessment *AssessmentConclusion `json:"assessment,omitempty"` +} + +// Validate checks ProjectionFacts' required UTC effective_started_at, the +// closed effective_started_at_basis/terminal_reason codes, every other +// captured instant's UTC-ness, the terminal_at/terminal_reason pairing, and +// the nested AssessmentConclusion when present. +func (p ProjectionFacts) Validate() error { + if p.PublicHandle != nil && strings.TrimSpace(*p.PublicHandle) == "" { + return errors.New("projection_facts: public_handle must not be empty when set") + } + if err := requireNonZeroUTC("effective_started_at", p.EffectiveStartedAt); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + if err := validateEnum("effective_started_at_basis", p.EffectiveStartedAtBasis, + SourceTimeBasisSourcePayload, SourceTimeBasisSourceAPI, SourceTimeBasisReceiptFallback, + SourceTimeBasisMissing, SourceTimeBasisMixed); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + if p.RecoveryObservedAt != nil { + if err := requireUTC("recovery_observed_at", *p.RecoveryObservedAt); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + } + if p.GraceUntil != nil { + if err := requireUTC("grace_until", *p.GraceUntil); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + } + if p.TerminalAt != nil { + if err := requireUTC("terminal_at", *p.TerminalAt); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + } + switch { + case p.TerminalAt != nil && p.TerminalReason == nil: + return errors.New("projection_facts: terminal_at requires terminal_reason") + case p.TerminalAt == nil && p.TerminalReason != nil: + return errors.New("projection_facts: terminal_reason requires terminal_at") + } + if p.TerminalReason != nil { + if err := validateEnum("terminal_reason", *p.TerminalReason, + TerminalReasonObservationDeadline, TerminalReasonResolutionMissing, + TerminalReasonSourceUnavailable, TerminalReasonBudgetExhausted); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + } + if p.Assessment != nil { + if err := p.Assessment.Validate(); err != nil { + return fmt.Errorf("projection_facts: %w", err) + } + } + return nil +} + +// ---------------------------------------------------------------------- +// Transition +// ---------------------------------------------------------------------- + +// Transition is the immutable record of one authoritative material change +// to a Situation: exactly one controller-state Transition per material +// reconciliation, plus one per newly journaled operator artifact (R1). It +// is never a copy of mutable current state and is never rewritten. +type Transition struct { + ID string `json:"id"` + SituationID string `json:"situation_id"` + Sequence int `json:"sequence"` + InputVersion int `json:"input_version"` + MaterialFactHash string `json:"material_fact_hash"` + AssessmentID *string `json:"assessment_id,omitempty"` + Lifecycle Lifecycle `json:"lifecycle"` + Attention Attention `json:"attention"` + ActionContract ActionContract `json:"action_contract"` + SufficientReasonID *string `json:"sufficient_reason_id,omitempty"` + InterruptionPriority *InterruptionPriority `json:"interruption_priority,omitempty"` + Reason TransitionReason `json:"reason"` + JournalKind JournalKind `json:"journal_kind"` + Journal JournalData `json:"journal"` + Projection ProjectionFacts `json:"projection"` // R3 + OperatorArtifactInputID *string `json:"operator_artifact_input_id,omitempty"` // R1: the consumed situation_input_outbox row + EvidenceRefs []string `json:"evidence_refs"` + Actor TransitionActor `json:"actor"` + Drill bool `json:"drill,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// MarshalJSON canonicalizes EvidenceRefs to [] before marshaling: a +// nil-constructed Transition must never serialize evidence_refs as JSON +// null. The nested ActionContract, Journal, and Projection canonicalize +// their own slice fields via their own MarshalJSON methods. +func (t Transition) MarshalJSON() ([]byte, error) { + type transitionAlias Transition + a := transitionAlias(t) + a.EvidenceRefs = canonicalizeSlice(a.EvidenceRefs) + return json.Marshal(a) +} + +// Validate checks Transition's required, bounded identity fields; its +// closed lifecycle/attention/reason/journal-kind/actor codes; the Operator +// contract's shape/consistency against the Transition's own Lifecycle; the +// nested Journal and Projection; the operator_artifact_recorded <-> +// OperatorArtifactInputID pairing (R1: required exactly for that reason, +// forbidden otherwise); the evidence-ref bound; and a required, UTC +// created_at. It is a shape/consistency check only — it never compares +// against a wall clock, since a historical Transition's own +// action_contract.next_update_at promise is expected to have long since +// elapsed by the time it is read back. +func (t Transition) Validate() error { + if strings.TrimSpace(t.ID) == "" { + return errors.New("transition: id is required") + } + if len(t.ID) > maxIdentifierLength { + return fmt.Errorf("transition: id exceeds %d bytes", maxIdentifierLength) + } + if strings.TrimSpace(t.SituationID) == "" { + return errors.New("transition: situation_id is required") + } + if t.Sequence < 1 { + return fmt.Errorf("transition: sequence must be >= 1, got %d", t.Sequence) + } + if t.InputVersion < 1 { + return fmt.Errorf("transition: input_version must be >= 1, got %d", t.InputVersion) + } + if strings.TrimSpace(t.MaterialFactHash) == "" { + return errors.New("transition: material_fact_hash is required") + } + if t.AssessmentID != nil && strings.TrimSpace(*t.AssessmentID) == "" { + return errors.New("transition: assessment_id must not be empty when set") + } + if err := t.Lifecycle.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + if err := t.Attention.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + if err := t.ActionContract.validate(t.Lifecycle.Terminal()); err != nil { + return fmt.Errorf("transition: %w", err) + } + if t.SufficientReasonID != nil && strings.TrimSpace(*t.SufficientReasonID) == "" { + return errors.New("transition: sufficient_reason_id must not be empty when set") + } + if t.InterruptionPriority != nil { + if err := t.InterruptionPriority.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + } + if err := t.Reason.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + if err := t.JournalKind.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + if err := t.Journal.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + if err := t.Projection.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + + switch { + case t.Reason == ReasonOperatorArtifactRecorded && t.OperatorArtifactInputID == nil: + return errors.New("transition: operator_artifact_input_id is required when reason is operator_artifact_recorded") + case t.Reason != ReasonOperatorArtifactRecorded && t.OperatorArtifactInputID != nil: + return errors.New("transition: operator_artifact_input_id must be unset unless reason is operator_artifact_recorded") + } + if t.OperatorArtifactInputID != nil && strings.TrimSpace(*t.OperatorArtifactInputID) == "" { + return errors.New("transition: operator_artifact_input_id must not be empty when set") + } + + if len(t.EvidenceRefs) > maxEvidenceRefs { + return fmt.Errorf("transition: evidence_refs exceeds %d entries", maxEvidenceRefs) + } + + if err := t.Actor.Validate(); err != nil { + return fmt.Errorf("transition: %w", err) + } + + if err := requireNonZeroUTC("created_at", t.CreatedAt); err != nil { + return fmt.Errorf("transition: %w", err) + } + return nil +} + +// ---------------------------------------------------------------------- +// Episode summary +// ---------------------------------------------------------------------- + +// EpisodeSummary is the current, versioned Episode-summary projection +// folded from a Situation's Transitions in sequence order — the durable +// content the Situation-owned Slack root renders. Every fold advances +// Version by exactly one. +type EpisodeSummary struct { + SituationID string `json:"situation_id"` + PublicHandle string `json:"public_handle,omitempty"` + Version int `json:"version"` + SourceTransitionSequence int `json:"source_transition_sequence"` + Title string `json:"title"` + InitialPublicationReason string `json:"initial_publication_reason,omitempty"` + LatestMaterialReason string `json:"latest_material_reason,omitempty"` + EvidenceConclusion string `json:"evidence_conclusion,omitempty"` + ImpactSummary string `json:"impact_summary,omitempty"` + InvestigationWork []string `json:"investigation_work"` + InvestigationStarted bool `json:"investigation_started"` + CurrentAttention Attention `json:"current_attention"` + PeakAttention Attention `json:"peak_attention"` + ActionContract ActionContract `json:"action_contract"` + RecordedOperatorContext []string `json:"recorded_operator_context"` + EffectiveStartedAt time.Time `json:"effective_started_at"` + RecoveryObservedAt *time.Time `json:"recovery_observed_at,omitempty"` + TerminalAt *time.Time `json:"terminal_at,omitempty"` + DurationSeconds *int64 `json:"duration_seconds,omitempty"` + RecurrenceCount int `json:"recurrence_count"` + FinalOutcome string `json:"final_outcome,omitempty"` + RemainingUncertainty string `json:"remaining_uncertainty,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// MarshalJSON canonicalizes InvestigationWork and RecordedOperatorContext +// to [] before marshaling: a nil-constructed EpisodeSummary must never +// serialize either as JSON null. The nested ActionContract canonicalizes +// its own slice field via its own MarshalJSON method. +func (e EpisodeSummary) MarshalJSON() ([]byte, error) { + type episodeSummaryAlias EpisodeSummary + out := episodeSummaryAlias(e) + out.InvestigationWork = canonicalizeSlice(out.InvestigationWork) + out.RecordedOperatorContext = canonicalizeSlice(out.RecordedOperatorContext) + return json.Marshal(out) +} + +// Validate checks EpisodeSummary's required identity, its closed +// current/peak Attention, the Operator contract's shape against whether the +// summary is terminal (terminal_at set), required UTC instants, and +// non-negative counters. Terminal-ness is derived from TerminalAt since +// EpisodeSummary carries no Lifecycle field of its own. +func (e EpisodeSummary) Validate() error { + if strings.TrimSpace(e.SituationID) == "" { + return errors.New("episode_summary: situation_id is required") + } + if e.Version < 1 { + return fmt.Errorf("episode_summary: version must be >= 1, got %d", e.Version) + } + if e.SourceTransitionSequence < 1 { + return fmt.Errorf("episode_summary: source_transition_sequence must be >= 1, got %d", e.SourceTransitionSequence) + } + if strings.TrimSpace(e.Title) == "" { + return errors.New("episode_summary: title is required") + } + if err := e.CurrentAttention.Validate(); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + if err := e.PeakAttention.Validate(); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + terminal := e.TerminalAt != nil + if err := e.ActionContract.validate(terminal); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + if err := requireNonZeroUTC("effective_started_at", e.EffectiveStartedAt); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + if e.RecoveryObservedAt != nil { + if err := requireUTC("recovery_observed_at", *e.RecoveryObservedAt); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + } + if e.TerminalAt != nil { + if err := requireUTC("terminal_at", *e.TerminalAt); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + } + if e.DurationSeconds != nil && *e.DurationSeconds < 0 { + return errors.New("episode_summary: duration_seconds must be >= 0") + } + if e.RecurrenceCount < 0 { + return errors.New("episode_summary: recurrence_count must be >= 0") + } + if err := requireNonZeroUTC("updated_at", e.UpdatedAt); err != nil { + return fmt.Errorf("episode_summary: %w", err) + } + return nil +} + +// ---------------------------------------------------------------------- +// Notification intent +// ---------------------------------------------------------------------- + +// EffectClass is the closed shape of one Slack notification effect. +type EffectClass string + +const ( + EffectRootSync EffectClass = "root_sync" + EffectThreadAppend EffectClass = "thread_append" + EffectBroadcastHandoff EffectClass = "broadcast_handoff" + EffectInstallationGapRecovery EffectClass = "installation_gap_recovery" +) + +// Validate reports an error unless e is one of the closed EffectClass +// values. +func (e EffectClass) Validate() error { + return validateEnum("effect_class", e, + EffectRootSync, EffectThreadAppend, EffectBroadcastHandoff, EffectInstallationGapRecovery) +} + +// IntentStatus is the closed lifecycle of one NotificationIntent. +type IntentStatus string + +const ( + IntentPending IntentStatus = "pending" + IntentDelivered IntentStatus = "delivered" + IntentBlockedConfiguration IntentStatus = "blocked_configuration" + IntentFailed IntentStatus = "failed" + IntentWithheld IntentStatus = "withheld_by_operator_slack_floor" + IntentSuperseded IntentStatus = "superseded" +) + +// Validate reports an error unless s is one of the closed IntentStatus +// values. +func (s IntentStatus) Validate() error { + return validateEnum("intent_status", s, + IntentPending, IntentDelivered, IntentBlockedConfiguration, IntentFailed, IntentWithheld, IntentSuperseded) +} + +// NotificationIntent is one durable, fenced Slack delivery obligation +// created inside the authoritative controller commit. It is never +// serialized with json struct tags: it is a store/worker-internal record, +// not a wire shape. +type NotificationIntent struct { + ID string + IdempotencyKey string + EffectClass EffectClass + SituationID *string + TransitionID *string + TransitionSequence *int + SummaryVersion *int + GapGeneration *string + RequiresRoot bool + MainChannelPoke bool + InterruptionPriority *InterruptionPriority + ContractDeadlineAt *time.Time // root_sync only (R4): the committed promise this root renders + ClientMessageID string + Status IntentStatus + ClaimOwner *string + ClaimToken int64 + LeaseExpiresAt *time.Time + AttemptCount int + LastErrorClass *string + RetryAt *time.Time + SupersessionReason *string + ReplacementIntentID *string + DeliveredAs *string + Channel *string + MessageTS *string + CreatedAt time.Time + DeliveredAt *time.Time +} + +// Validate checks NotificationIntent's required, bounded identity fields; +// its closed effect-class/status/priority codes; the effect-class-specific +// reference rules (installation_gap_recovery forbids Situation/Transition +// references and requires a gap generation; the three Situation effects +// require a Situation/Transition/summary reference according to their +// class; contract_deadline_at is accepted only on root_sync; gap_generation +// is accepted only on installation_gap_recovery); and a required, UTC +// created_at. +func (n NotificationIntent) Validate() error { + if strings.TrimSpace(n.ID) == "" { + return errors.New("notification_intent: id is required") + } + if len(n.ID) > maxIdentifierLength { + return fmt.Errorf("notification_intent: id exceeds %d bytes", maxIdentifierLength) + } + if strings.TrimSpace(n.IdempotencyKey) == "" { + return errors.New("notification_intent: idempotency_key is required") + } + if len(n.IdempotencyKey) > maxIdentifierLength { + return fmt.Errorf("notification_intent: idempotency_key exceeds %d bytes", maxIdentifierLength) + } + if err := n.EffectClass.Validate(); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + if strings.TrimSpace(n.ClientMessageID) == "" { + return errors.New("notification_intent: client_message_id is required") + } + if len(n.ClientMessageID) > maxIdentifierLength { + return fmt.Errorf("notification_intent: client_message_id exceeds %d bytes", maxIdentifierLength) + } + if err := n.Status.Validate(); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + if n.InterruptionPriority != nil { + if err := n.InterruptionPriority.Validate(); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + } + if n.AttemptCount < 0 { + return errors.New("notification_intent: attempt_count must be >= 0") + } + for _, ptrField := range []struct { + name string + v *string + }{ + {"situation_id", n.SituationID}, {"transition_id", n.TransitionID}, + {"gap_generation", n.GapGeneration}, {"claim_owner", n.ClaimOwner}, + {"last_error_class", n.LastErrorClass}, {"supersession_reason", n.SupersessionReason}, + {"replacement_intent_id", n.ReplacementIntentID}, {"delivered_as", n.DeliveredAs}, + {"channel", n.Channel}, {"message_ts", n.MessageTS}, + } { + if ptrField.v == nil { + continue + } + if strings.TrimSpace(*ptrField.v) == "" { + return fmt.Errorf("notification_intent: %s must not be empty when set", ptrField.name) + } + if len(*ptrField.v) > maxIdentifierLength { + return fmt.Errorf("notification_intent: %s exceeds %d bytes", ptrField.name, maxIdentifierLength) + } + } + + switch n.EffectClass { + case EffectInstallationGapRecovery: + if n.SituationID != nil { + return errors.New("notification_intent: installation_gap_recovery must not set situation_id") + } + if n.TransitionID != nil { + return errors.New("notification_intent: installation_gap_recovery must not set transition_id") + } + if n.TransitionSequence != nil { + return errors.New("notification_intent: installation_gap_recovery must not set transition_sequence") + } + if n.SummaryVersion != nil { + return errors.New("notification_intent: installation_gap_recovery must not set summary_version") + } + if n.GapGeneration == nil { + return errors.New("notification_intent: installation_gap_recovery requires gap_generation") + } + case EffectRootSync, EffectThreadAppend, EffectBroadcastHandoff: + if n.SituationID == nil { + return fmt.Errorf("notification_intent: %s requires situation_id", n.EffectClass) + } + if n.TransitionID == nil { + return fmt.Errorf("notification_intent: %s requires transition_id", n.EffectClass) + } + if n.GapGeneration != nil { + return fmt.Errorf("notification_intent: gap_generation is accepted only on %s, not %s", EffectInstallationGapRecovery, n.EffectClass) + } + if n.EffectClass == EffectRootSync && n.SummaryVersion == nil { + return errors.New("notification_intent: root_sync requires summary_version") + } + } + + if n.ContractDeadlineAt != nil { + if n.EffectClass != EffectRootSync { + return fmt.Errorf("notification_intent: contract_deadline_at is accepted only on %s, not %s", EffectRootSync, n.EffectClass) + } + if err := requireUTC("contract_deadline_at", *n.ContractDeadlineAt); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + } + + if err := requireNonZeroUTC("created_at", n.CreatedAt); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + if n.LeaseExpiresAt != nil { + if err := requireUTC("lease_expires_at", *n.LeaseExpiresAt); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + } + if n.RetryAt != nil { + if err := requireUTC("retry_at", *n.RetryAt); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + } + if n.DeliveredAt != nil { + if err := requireUTC("delivered_at", *n.DeliveredAt); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + } + return nil +} diff --git a/internal/situation/model/history_test.go b/internal/situation/model/history_test.go new file mode 100644 index 0000000..2cf735c --- /dev/null +++ b/internal/situation/model/history_test.go @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package model + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// ptr returns a pointer to a copy of v. Kept distinct from assessment_test.go's +// samplePointer so history_test.go's fixtures read close to the plan's own +// example test code. +func ptr[T any](v T) *T { return &v } + +// ---------------------------------------------------------------------- +// Closed enums +// ---------------------------------------------------------------------- + +// TestTransitionRelatedEnumsValidate table-tests every closed value for the +// enums this file defines (except TransitionActor, which has its own test +// below since Plan 3 accepts only three of its four defined values). +func TestTransitionRelatedEnumsValidate(t *testing.T) { + cases := []struct { + name string + valid []enumValidator + bogus enumValidator + }{ + { + name: "TransitionReason", + valid: []enumValidator{ + ReasonFirstAuthoritativeState, ReasonMaterialAssessmentChanged, ReasonAttentionChanged, + ReasonOperatorContractChanged, ReasonInvestigationStarted, ReasonInvestigationConcluded, + ReasonRecoveryObserved, ReasonRecoveryFailed, ReasonRecovered, ReasonClosedUnknown, + ReasonRecurrenceMilestone, ReasonTriageStateChanged, ReasonOperatorArtifactRecorded, + }, + bogus: TransitionReason("bogus"), + }, + { + name: "JournalKind", + valid: []enumValidator{ + JournalNone, JournalPublication, JournalInvestigationStarted, JournalInvestigationChanged, + JournalEvidenceConclusion, JournalOperatorContractChanged, JournalRecoveryPending, + JournalRecoveryRefired, JournalRecurrenceMilestone, JournalRecovered, JournalClosedUnknown, + JournalOperatorNote, JournalCapturedVerdict, + }, + bogus: JournalKind("bogus"), + }, + { + name: "EffectClass", + valid: []enumValidator{EffectRootSync, EffectThreadAppend, EffectBroadcastHandoff, EffectInstallationGapRecovery}, + bogus: EffectClass("bogus"), + }, + { + name: "IntentStatus", + valid: []enumValidator{IntentPending, IntentDelivered, IntentBlockedConfiguration, IntentFailed, IntentWithheld, IntentSuperseded}, + bogus: IntentStatus("bogus"), + }, + { + name: "InterruptionPriority", + valid: []enumValidator{InterruptionLow, InterruptionMedium, InterruptionHigh, InterruptionCritical}, + bogus: InterruptionPriority("bogus"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, v := range tc.valid { + if err := v.Validate(); err != nil { + t.Errorf("valid value %v: unexpected error: %v", v, err) + } + } + if err := tc.bogus.Validate(); err == nil { + t.Errorf("bogus value %v: want error, got nil", tc.bogus) + } + }) + } +} + +// TestTransitionActorValidate proves Plan 3 accepts exactly +// deterministic_controller, llm, and attributed_operator, and rejects +// operator_policy even though it is a defined TransitionActor constant +// (reserved for Plan 5; spec.md "Domain model"). +func TestTransitionActorValidate(t *testing.T) { + for _, a := range []TransitionActor{ActorDeterministicController, ActorLLM, ActorAttributedOperator} { + if err := a.Validate(); err != nil { + t.Errorf("valid actor %q: unexpected error: %v", a, err) + } + } + if err := ActorOperatorPolicy.Validate(); err == nil { + t.Error("operator_policy: want error in Plan 3, got nil") + } + if err := TransitionActor("bogus").Validate(); err == nil { + t.Error("bogus actor: want error, got nil") + } +} + +// TestInterruptionPriorityOrdering proves the deterministic ranking +// low < medium < high < critical. +func TestInterruptionPriorityOrdering(t *testing.T) { + order := []InterruptionPriority{InterruptionLow, InterruptionMedium, InterruptionHigh, InterruptionCritical} + for i := range order { + for j := range order { + want := i < j + if got := order[i].Less(order[j]); got != want { + t.Errorf("%s.Less(%s) = %v, want %v", order[i], order[j], got, want) + } + } + } +} + +// ---------------------------------------------------------------------- +// JournalData +// ---------------------------------------------------------------------- + +func fullJournalData(now time.Time) JournalData { + return JournalData{ + Headline: "situation opened", + Detail: "first authoritative state derived from incident_created", + AttributedActor: "alice", + ActionStatus: "planned", + RecurrenceCount: 2, + Delayed: false, + NoLongerCurrent: false, + OccurredAt: now, + } +} + +func TestJournalDataJSONRoundTrip(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + before, after := roundTrip(t, fullJournalData(now)) + if string(before) != string(after) { + t.Fatalf("round trip not lossless:\nbefore: %s\nafter: %s", before, after) + } +} + +func TestJournalDataValidate(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + base := fullJournalData(now) + if err := base.Validate(); err != nil { + t.Fatalf("valid journal data: unexpected error: %v", err) + } + + cases := []struct { + name string + mutate func(*JournalData) + }{ + {"headline over max length", func(j *JournalData) { j.Headline = strings.Repeat("a", maxJournalHeadlineLength+1) }}, + {"detail over max length", func(j *JournalData) { j.Detail = strings.Repeat("a", maxJournalDetailLength+1) }}, + {"attributed_actor over max length", func(j *JournalData) { j.AttributedActor = strings.Repeat("a", maxIdentifierLength+1) }}, + {"action_status over max length", func(j *JournalData) { j.ActionStatus = strings.Repeat("a", maxIdentifierLength+1) }}, + {"negative recurrence_count", func(j *JournalData) { j.RecurrenceCount = -1 }}, + {"zero occurred_at", func(j *JournalData) { j.OccurredAt = time.Time{} }}, + {"non-UTC occurred_at", func(j *JournalData) { j.OccurredAt = now.In(time.FixedZone("test", 3600)) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + j := fullJournalData(now) + tc.mutate(&j) + if err := j.Validate(); err == nil { + t.Fatalf("%s: want error, got nil", tc.name) + } + }) + } +} + +// ---------------------------------------------------------------------- +// AssessmentConclusion / ProjectionFacts (R3) +// ---------------------------------------------------------------------- + +func fullAssessmentConclusion() AssessmentConclusion { + return AssessmentConclusion{ + Persistence: PersistenceSustained, + Impact: ImpactConfirmed, + Novelty: NoveltyChanged, + Causality: CausalitySupported, + EvidenceQuality: EvidenceQualityComplete, + LimitationCodes: []string{"semantic_assessment_unavailable"}, + SufficientReasonCode: "duration_milestone", + SufficientReasonSummary: "sustained beyond the milestone threshold", + } +} + +func TestAssessmentConclusionJSONRoundTrip(t *testing.T) { + before, after := roundTrip(t, fullAssessmentConclusion()) + if string(before) != string(after) { + t.Fatalf("round trip not lossless:\nbefore: %s\nafter: %s", before, after) + } +} + +func TestAssessmentConclusionCanonicalizesNilLimitationCodes(t *testing.T) { + c := fullAssessmentConclusion() + c.LimitationCodes = nil + b, err := json.Marshal(c) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(b), `"limitation_codes":[]`) { + t.Errorf("want canonical empty limitation_codes array, got %s", b) + } +} + +func TestAssessmentConclusionValidate(t *testing.T) { + valid := fullAssessmentConclusion() + if err := valid.Validate(); err != nil { + t.Fatalf("valid assessment conclusion: unexpected error: %v", err) + } + + cases := []struct { + name string + mutate func(*AssessmentConclusion) + }{ + {"bad persistence", func(c *AssessmentConclusion) { c.Persistence = Persistence("bogus") }}, + {"bad impact", func(c *AssessmentConclusion) { c.Impact = Impact("bogus") }}, + {"bad novelty", func(c *AssessmentConclusion) { c.Novelty = Novelty("bogus") }}, + {"bad causality", func(c *AssessmentConclusion) { c.Causality = Causality("bogus") }}, + {"bad evidence_quality", func(c *AssessmentConclusion) { c.EvidenceQuality = EvidenceQuality("bogus") }}, + {"sufficient_reason_summary over max length", func(c *AssessmentConclusion) { + c.SufficientReasonSummary = strings.Repeat("a", maxJournalDetailLength+1) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := fullAssessmentConclusion() + tc.mutate(&c) + if err := c.Validate(); err == nil { + t.Fatalf("%s: want error, got nil", tc.name) + } + }) + } +} + +func fullProjectionFacts(now time.Time) ProjectionFacts { + assessment := fullAssessmentConclusion() + return ProjectionFacts{ + PublicHandle: ptr("sit-abc123"), + EffectiveStartedAt: now.Add(-time.Hour), + EffectiveStartedAtBasis: SourceTimeBasisSourcePayload, + Assessment: &assessment, + } +} + +func TestProjectionFactsJSONRoundTrip(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + before, after := roundTrip(t, fullProjectionFacts(now)) + if string(before) != string(after) { + t.Fatalf("round trip not lossless:\nbefore: %s\nafter: %s", before, after) + } +} + +func TestProjectionFactsValidate(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + valid := fullProjectionFacts(now) + if err := valid.Validate(); err != nil { + t.Fatalf("valid projection facts: unexpected error: %v", err) + } + + t.Run("zero effective_started_at rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.EffectiveStartedAt = time.Time{} + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("non-UTC effective_started_at rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.EffectiveStartedAt = now.In(time.FixedZone("test", 3600)) + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("bad effective_started_at_basis rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.EffectiveStartedAtBasis = SourceTimeBasis("bogus") + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("empty public_handle rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.PublicHandle = ptr("") + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("non-UTC recovery_observed_at rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.RecoveryObservedAt = ptr(now.In(time.FixedZone("test", 3600))) + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("terminal_at without terminal_reason rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.TerminalAt = ptr(now) + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("terminal_reason without terminal_at rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.TerminalReason = ptr(TerminalReasonObservationDeadline) + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("terminal_at with terminal_reason accepted", func(t *testing.T) { + p := fullProjectionFacts(now) + p.TerminalAt = ptr(now) + p.TerminalReason = ptr(TerminalReasonObservationDeadline) + if err := p.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("bad terminal_reason rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + p.TerminalAt = ptr(now) + bogus := TerminalReason("bogus") + p.TerminalReason = &bogus + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("invalid nested assessment rejected", func(t *testing.T) { + p := fullProjectionFacts(now) + bad := fullAssessmentConclusion() + bad.Persistence = Persistence("bogus") + p.Assessment = &bad + if err := p.Validate(); err == nil { + t.Fatal("want error, got nil") + } + }) +} + +// ---------------------------------------------------------------------- +// Transition +// ---------------------------------------------------------------------- + +func fullTransition(now time.Time) Transition { + priority := InterruptionHigh + return Transition{ + ID: "transition-1", + SituationID: "situation-1", + Sequence: 1, + InputVersion: 1, + MaterialFactHash: "hash-1", + AssessmentID: ptr("assessment-1"), + Lifecycle: LifecycleActive, + Attention: AttentionUrgent, + ActionContract: fullActionContract(now), + SufficientReasonID: ptr("reason-1"), + InterruptionPriority: &priority, + Reason: ReasonFirstAuthoritativeState, + JournalKind: JournalPublication, + Journal: fullJournalData(now), + Projection: fullProjectionFacts(now), + EvidenceRefs: []string{"fact-1"}, + Actor: ActorDeterministicController, + CreatedAt: now, + } +} + +func TestTransitionJSONRoundTrip(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + before, after := roundTrip(t, fullTransition(now)) + if string(before) != string(after) { + t.Fatalf("round trip not lossless:\nbefore: %s\nafter: %s", before, after) + } +} + +func TestTransitionCanonicalizesNilEvidenceRefs(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + tr := fullTransition(now) + tr.EvidenceRefs = nil + b, err := json.Marshal(tr) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(b), `"evidence_refs":[]`) { + t.Errorf("want canonical empty evidence_refs array, got %s", b) + } +} + +func TestTransitionValidate(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + valid := fullTransition(now) + if err := valid.Validate(); err != nil { + t.Fatalf("valid transition: unexpected error: %v", err) + } + + cases := []struct { + name string + mutate func(*Transition) + }{ + {"empty id", func(tr *Transition) { tr.ID = "" }}, + {"id over max length", func(tr *Transition) { tr.ID = strings.Repeat("a", maxIdentifierLength+1) }}, + {"empty situation_id", func(tr *Transition) { tr.SituationID = "" }}, + {"sequence zero", func(tr *Transition) { tr.Sequence = 0 }}, + {"input_version zero", func(tr *Transition) { tr.InputVersion = 0 }}, + {"empty material_fact_hash", func(tr *Transition) { tr.MaterialFactHash = "" }}, + {"empty assessment_id when set", func(tr *Transition) { tr.AssessmentID = ptr("") }}, + {"bad lifecycle", func(tr *Transition) { tr.Lifecycle = Lifecycle("bogus") }}, + {"bad attention", func(tr *Transition) { tr.Attention = Attention("bogus") }}, + {"empty sufficient_reason_id when set", func(tr *Transition) { tr.SufficientReasonID = ptr("") }}, + {"bad interruption_priority", func(tr *Transition) { tr.InterruptionPriority = ptr(InterruptionPriority("bogus")) }}, + {"bad reason", func(tr *Transition) { tr.Reason = TransitionReason("bogus") }}, + {"bad journal_kind", func(tr *Transition) { tr.JournalKind = JournalKind("bogus") }}, + {"invalid journal", func(tr *Transition) { tr.Journal.OccurredAt = time.Time{} }}, + {"invalid projection", func(tr *Transition) { tr.Projection.EffectiveStartedAt = time.Time{} }}, + {"operator_policy actor rejected", func(tr *Transition) { tr.Actor = ActorOperatorPolicy }}, + {"bad actor", func(tr *Transition) { tr.Actor = TransitionActor("bogus") }}, + {"evidence_refs over max count", func(tr *Transition) { + refs := make([]string, maxEvidenceRefs+1) + for i := range refs { + refs[i] = "ref" + } + tr.EvidenceRefs = refs + }}, + {"zero created_at", func(tr *Transition) { tr.CreatedAt = time.Time{} }}, + {"non-UTC created_at", func(tr *Transition) { tr.CreatedAt = now.In(time.FixedZone("test", 3600)) }}, + {"terminal contract shape violated", func(tr *Transition) { tr.Lifecycle = LifecycleRecovered }}, + {"operator_artifact_recorded without input id", func(tr *Transition) { + tr.Reason = ReasonOperatorArtifactRecorded + tr.OperatorArtifactInputID = nil + }}, + {"non-artifact reason with input id set", func(tr *Transition) { + tr.OperatorArtifactInputID = ptr("input-1") + }}, + {"empty operator_artifact_input_id when set", func(tr *Transition) { + tr.Reason = ReasonOperatorArtifactRecorded + tr.OperatorArtifactInputID = ptr("") + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tr := fullTransition(now) + tc.mutate(&tr) + if err := tr.Validate(); err == nil { + t.Fatalf("%s: want error, got nil", tc.name) + } + }) + } +} + +// TestTransitionValidateOperatorArtifactRecordedRequiresInputID proves R1's +// exact pairing: operator_artifact_input_id is required when (and only +// when) reason is operator_artifact_recorded. +func TestTransitionValidateOperatorArtifactRecordedRequiresInputID(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + tr := fullTransition(now) + tr.Reason = ReasonOperatorArtifactRecorded + tr.OperatorArtifactInputID = ptr("input-1") + if err := tr.Validate(); err != nil { + t.Fatalf("operator_artifact_recorded with input id: unexpected error: %v", err) + } +} + +// ---------------------------------------------------------------------- +// EpisodeSummary +// ---------------------------------------------------------------------- + +func fullEpisodeSummary(now time.Time) EpisodeSummary { + return EpisodeSummary{ + SituationID: "situation-1", + PublicHandle: "sit-abc123", + Version: 1, + SourceTransitionSequence: 1, + Title: "database connection pool exhausted", + InitialPublicationReason: "first_authoritative_state", + LatestMaterialReason: "material_assessment_changed", + EvidenceConclusion: "confirmed via connection pool metrics", + ImpactSummary: "elevated latency on checkout", + InvestigationWork: []string{"checked connection pool metrics"}, + InvestigationStarted: true, + CurrentAttention: AttentionInvestigate, + PeakAttention: AttentionUrgent, + ActionContract: fullActionContract(now), + RecordedOperatorContext: []string{"operator confirmed deploy rollback"}, + EffectiveStartedAt: now.Add(-time.Hour), + RecurrenceCount: 0, + UpdatedAt: now, + } +} + +func TestEpisodeSummaryJSONRoundTrip(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + before, after := roundTrip(t, fullEpisodeSummary(now)) + if string(before) != string(after) { + t.Fatalf("round trip not lossless:\nbefore: %s\nafter: %s", before, after) + } +} + +func TestEpisodeSummaryCanonicalizesNilSlices(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + e := fullEpisodeSummary(now) + e.InvestigationWork = nil + e.RecordedOperatorContext = nil + b, err := json.Marshal(e) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(b), `"investigation_work":[]`) { + t.Errorf("want canonical empty investigation_work array, got %s", b) + } + if !strings.Contains(string(b), `"recorded_operator_context":[]`) { + t.Errorf("want canonical empty recorded_operator_context array, got %s", b) + } +} + +func TestEpisodeSummaryValidate(t *testing.T) { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + valid := fullEpisodeSummary(now) + if err := valid.Validate(); err != nil { + t.Fatalf("valid episode summary: unexpected error: %v", err) + } + + cases := []struct { + name string + mutate func(*EpisodeSummary) + }{ + {"empty situation_id", func(e *EpisodeSummary) { e.SituationID = "" }}, + {"version zero", func(e *EpisodeSummary) { e.Version = 0 }}, + {"source_transition_sequence zero", func(e *EpisodeSummary) { e.SourceTransitionSequence = 0 }}, + {"empty title", func(e *EpisodeSummary) { e.Title = "" }}, + {"bad current_attention", func(e *EpisodeSummary) { e.CurrentAttention = Attention("bogus") }}, + {"bad peak_attention", func(e *EpisodeSummary) { e.PeakAttention = Attention("bogus") }}, + {"zero effective_started_at", func(e *EpisodeSummary) { e.EffectiveStartedAt = time.Time{} }}, + {"non-UTC effective_started_at", func(e *EpisodeSummary) { e.EffectiveStartedAt = now.In(time.FixedZone("test", 3600)) }}, + {"non-UTC recovery_observed_at", func(e *EpisodeSummary) { e.RecoveryObservedAt = ptr(now.In(time.FixedZone("test", 3600))) }}, + {"negative duration_seconds", func(e *EpisodeSummary) { e.DurationSeconds = ptr(int64(-1)) }}, + {"negative recurrence_count", func(e *EpisodeSummary) { e.RecurrenceCount = -1 }}, + {"zero updated_at", func(e *EpisodeSummary) { e.UpdatedAt = time.Time{} }}, + {"non-UTC updated_at", func(e *EpisodeSummary) { e.UpdatedAt = now.In(time.FixedZone("test", 3600)) }}, + {"terminal contract shape violated", func(e *EpisodeSummary) { e.TerminalAt = ptr(now) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e := fullEpisodeSummary(now) + tc.mutate(&e) + if err := e.Validate(); err == nil { + t.Fatalf("%s: want error, got nil", tc.name) + } + }) + } +} + +// ---------------------------------------------------------------------- +// NotificationIntent +// ---------------------------------------------------------------------- + +func validIntent() NotificationIntent { + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + return NotificationIntent{ + ID: "intent-1", + IdempotencyKey: "situation-1:root_sync:summary-3", + EffectClass: EffectRootSync, + SituationID: ptr("situation-1"), + TransitionID: ptr("transition-1"), + SummaryVersion: ptr(3), + ClientMessageID: "11111111-1111-1111-1111-111111111111", + Status: IntentPending, + CreatedAt: now, + } +} + +// TestNotificationIntentValidateGapRecovery is the plan's binding example: +// installation_gap_recovery with no Situation/Transition reference and a +// gap generation validates cleanly. +func TestNotificationIntentValidateGapRecovery(t *testing.T) { + in := validIntent() + in.EffectClass = EffectInstallationGapRecovery + in.SituationID, in.TransitionID = nil, nil + in.SummaryVersion = nil + in.GapGeneration = ptr("gap-7") + if err := in.Validate(); err != nil { + t.Fatal(err) + } +} + +// TestNotificationIntentValidateEffectClassRequirements proves the +// effect-class-specific reference rules: installation_gap_recovery forbids +// Situation/Transition references and requires a gap generation; the three +// Situation effects each require a Situation/Transition (and, for +// root_sync, a summary version) reference; gap_generation is accepted only +// on installation_gap_recovery; contract_deadline_at is accepted only on +// root_sync. +func TestNotificationIntentValidateEffectClassRequirements(t *testing.T) { + deadline := time.Date(2026, 9, 5, 13, 0, 0, 0, time.UTC) + + cases := []struct { + name string + mutate func(*NotificationIntent) + wantErr bool + }{ + {"root_sync valid", func(n *NotificationIntent) {}, false}, + {"root_sync missing situation_id", func(n *NotificationIntent) { n.SituationID = nil }, true}, + {"root_sync missing transition_id", func(n *NotificationIntent) { n.TransitionID = nil }, true}, + {"root_sync missing summary_version", func(n *NotificationIntent) { n.SummaryVersion = nil }, true}, + {"root_sync with gap_generation forbidden", func(n *NotificationIntent) { n.GapGeneration = ptr("gap-1") }, true}, + {"root_sync with contract_deadline_at accepted", func(n *NotificationIntent) { n.ContractDeadlineAt = &deadline }, false}, + + {"thread_append valid", func(n *NotificationIntent) { + n.EffectClass = EffectThreadAppend + n.SummaryVersion = nil + }, false}, + {"thread_append missing situation_id", func(n *NotificationIntent) { + n.EffectClass = EffectThreadAppend + n.SummaryVersion = nil + n.SituationID = nil + }, true}, + {"thread_append missing transition_id", func(n *NotificationIntent) { + n.EffectClass = EffectThreadAppend + n.SummaryVersion = nil + n.TransitionID = nil + }, true}, + {"thread_append with contract_deadline_at forbidden", func(n *NotificationIntent) { + n.EffectClass = EffectThreadAppend + n.SummaryVersion = nil + n.ContractDeadlineAt = &deadline + }, true}, + + {"broadcast_handoff valid", func(n *NotificationIntent) { + n.EffectClass = EffectBroadcastHandoff + n.SummaryVersion = nil + }, false}, + {"broadcast_handoff missing transition_id", func(n *NotificationIntent) { + n.EffectClass = EffectBroadcastHandoff + n.SummaryVersion = nil + n.TransitionID = nil + }, true}, + + {"installation_gap_recovery valid", func(n *NotificationIntent) { + n.EffectClass = EffectInstallationGapRecovery + n.SituationID, n.TransitionID, n.SummaryVersion = nil, nil, nil + n.GapGeneration = ptr("gap-1") + }, false}, + {"installation_gap_recovery forbids situation_id", func(n *NotificationIntent) { + n.EffectClass = EffectInstallationGapRecovery + n.TransitionID, n.SummaryVersion = nil, nil + n.GapGeneration = ptr("gap-1") + }, true}, + {"installation_gap_recovery forbids transition_id", func(n *NotificationIntent) { + n.EffectClass = EffectInstallationGapRecovery + n.SituationID, n.SummaryVersion = nil, nil + n.GapGeneration = ptr("gap-1") + }, true}, + {"installation_gap_recovery forbids summary_version", func(n *NotificationIntent) { + n.EffectClass = EffectInstallationGapRecovery + n.SituationID, n.TransitionID = nil, nil + n.GapGeneration = ptr("gap-1") + }, true}, + {"installation_gap_recovery requires gap_generation", func(n *NotificationIntent) { + n.EffectClass = EffectInstallationGapRecovery + n.SituationID, n.TransitionID, n.SummaryVersion = nil, nil, nil + }, true}, + {"installation_gap_recovery with contract_deadline_at forbidden", func(n *NotificationIntent) { + n.EffectClass = EffectInstallationGapRecovery + n.SituationID, n.TransitionID, n.SummaryVersion = nil, nil, nil + n.GapGeneration = ptr("gap-1") + n.ContractDeadlineAt = &deadline + }, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := validIntent() + tc.mutate(&n) + err := n.Validate() + if tc.wantErr && err == nil { + t.Fatal("want error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("want no error, got %v", err) + } + }) + } +} + +func TestNotificationIntentValidateRequiresIdentity(t *testing.T) { + cases := []struct { + name string + mutate func(*NotificationIntent) + }{ + {"empty id", func(n *NotificationIntent) { n.ID = "" }}, + {"id over max length", func(n *NotificationIntent) { n.ID = strings.Repeat("a", maxIdentifierLength+1) }}, + {"empty idempotency_key", func(n *NotificationIntent) { n.IdempotencyKey = "" }}, + {"empty client_message_id", func(n *NotificationIntent) { n.ClientMessageID = "" }}, + {"bad effect_class", func(n *NotificationIntent) { n.EffectClass = EffectClass("bogus") }}, + {"bad status", func(n *NotificationIntent) { n.Status = IntentStatus("bogus") }}, + {"zero created_at", func(n *NotificationIntent) { n.CreatedAt = time.Time{} }}, + {"non-UTC created_at", func(n *NotificationIntent) { n.CreatedAt = n.CreatedAt.In(time.FixedZone("test", 3600)) }}, + {"negative attempt_count", func(n *NotificationIntent) { n.AttemptCount = -1 }}, + {"bad interruption_priority", func(n *NotificationIntent) { n.InterruptionPriority = ptr(InterruptionPriority("bogus")) }}, + {"empty situation_id when set", func(n *NotificationIntent) { n.SituationID = ptr("") }}, + {"empty claim_owner when set", func(n *NotificationIntent) { n.ClaimOwner = ptr("") }}, + {"empty last_error_class when set", func(n *NotificationIntent) { n.LastErrorClass = ptr("") }}, + {"non-UTC lease_expires_at", func(n *NotificationIntent) { + n.LeaseExpiresAt = ptr(n.CreatedAt.In(time.FixedZone("test", 3600))) + }}, + {"non-UTC delivered_at", func(n *NotificationIntent) { + n.DeliveredAt = ptr(n.CreatedAt.In(time.FixedZone("test", 3600))) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := validIntent() + tc.mutate(&n) + if err := n.Validate(); err == nil { + t.Fatalf("%s: want error, got nil", tc.name) + } + }) + } +} diff --git a/internal/situation/model/model.go b/internal/situation/model/model.go index 0204553..5068556 100644 --- a/internal/situation/model/model.go +++ b/internal/situation/model/model.go @@ -119,8 +119,27 @@ const ( DueObservationDeadline DueReason = "observation_deadline" DueRetry DueReason = "retry_due" DueUpgradeReconstruction DueReason = "upgrade_reconstruction" + // DueOperatorArtifactRecorded marks a Situation due because a durable + // operator artifact input (an attributed annotation or a Captured + // verdict) was applied and awaits journaling (R5). It is distinct from + // DueOperatorJudgment, which stays reserved for Plan 5's steering + // catalog — an annotation or verdict must never read as a judgment. + DueOperatorArtifactRecorded DueReason = "operator_artifact_recorded" ) +// Validate reports an error unless d is one of the closed DueReason values. +// The SQL due_reasons_json CHECK on situations (migration 0014) is a +// JSON-array-only check, so this validator is the actual gate on DueReason's +// closed vocabulary. +func (d DueReason) Validate() error { + return validateEnum("due_reason", d, + DueIncidentCreated, DueMembershipChanged, DueNewSymptom, DueAlertResolved, DueAlertRefired, + DueDurationMilestone, DueConnectorHealthChanged, DueSemanticProfileChanged, DueTriageChanged, + DueOperatorJudgment, DueEnvelopeChanged, DueEnvelopeBoundary, DueJudgmentBoundary, + DueManualReassessment, DueRecoveryGraceExpired, DueObservationDeadline, DueRetry, + DueUpgradeReconstruction, DueOperatorArtifactRecorded) +} + // TerminalReason is the structured reason recorded when a Situation closes // as closed_unknown. type TerminalReason string diff --git a/internal/situation/model/model_test.go b/internal/situation/model/model_test.go new file mode 100644 index 0000000..d5974d2 --- /dev/null +++ b/internal/situation/model/model_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package model + +import "testing" + +// TestDueReasonValidate proves DueReason's closed set accepts Plan 2's +// eighteen values plus R5's DueOperatorArtifactRecorded, and rejects the +// near-miss "operator_artifact" and other unknown strings. The +// due_reasons_json SQL CHECK (internal/store/migrations/0014_situation_foundation.sql) +// only checks that the column is a JSON array — this validator is the +// actual gate on the closed value vocabulary. +func TestDueReasonValidate(t *testing.T) { + valid := []DueReason{ + DueIncidentCreated, DueMembershipChanged, DueNewSymptom, DueAlertResolved, DueAlertRefired, + DueDurationMilestone, DueConnectorHealthChanged, DueSemanticProfileChanged, DueTriageChanged, + DueOperatorJudgment, DueEnvelopeChanged, DueEnvelopeBoundary, DueJudgmentBoundary, + DueManualReassessment, DueRecoveryGraceExpired, DueObservationDeadline, DueRetry, + DueUpgradeReconstruction, DueOperatorArtifactRecorded, + } + if len(valid) != 19 { + t.Fatalf("expected 19 closed DueReason values (Plan 2's 18 + R5), got %d", len(valid)) + } + for _, v := range valid { + if err := v.Validate(); err != nil { + t.Errorf("valid due reason %q: unexpected error: %v", v, err) + } + } + + bogus := []DueReason{"operator_artifact", "operator_judgment_needed", "bogus", ""} + for _, v := range bogus { + if err := v.Validate(); err == nil { + t.Errorf("bogus due reason %q: want error, got nil", v) + } + } +} From ae9f33ffd4fbe3ef6a921d799c453e87be52c90a Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 00:05:53 +0300 Subject: [PATCH 02/31] feat(store): add immutable Situation history schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 0017 adds Plan 3's history schema: the immutable situation_transitions ledger, situations' current-Transition pointer/sequence, the one-row-per-Situation situation_episode_summaries projection with its monotonic version/source-sequence fence, the situation_transition_stream stdout outbox, and a create-copy-drop-rename rebuild of situation_input_outbox adding exact operator-artifact input provenance (annotation_id/verdict_id) and the R1/R2 journaling cursor (journal_state, journaled_transition_id). Every Plan 1/2 outbox row, index, and CHECK is preserved verbatim; no historical Transition is fabricated for a pre-Plan-3 Situation. ApplySituationInput now stamps the exact applied_input_version on every apply, maps both artifact kinds to DueOperatorArtifactRecorded (R5), and implements R2: an artifact input that reaches an already-terminal owner is recorded (journal_state='owner_terminal') without joining — no input_version bump, no due reason, no lease clear — while every other kind keeps Plan 2's join/create behaviour unchanged. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- .../migrations/0017_situation_history.sql | 251 ++++++ .../store/situation_history_upgrade_test.go | 793 ++++++++++++++++++ internal/store/situations.go | 161 +++- internal/store/situations_test.go | 283 ++++++- internal/store/store_test.go | 15 +- 5 files changed, 1460 insertions(+), 43 deletions(-) create mode 100644 internal/store/migrations/0017_situation_history.sql create mode 100644 internal/store/situation_history_upgrade_test.go diff --git a/internal/store/migrations/0017_situation_history.sql b/internal/store/migrations/0017_situation_history.sql new file mode 100644 index 0000000..bfa8ff8 --- /dev/null +++ b/internal/store/migrations/0017_situation_history.sql @@ -0,0 +1,251 @@ +-- SPDX-License-Identifier: FSL-1.1-ALv2 +-- +-- Plan 3 history schema: the immutable Transition ledger, the current +-- Episode-summary projection, the stdout transition-stream outbox, and the +-- situation_input_outbox rebuild that adds exact operator-artifact input +-- provenance and the R1/R2 journaling cursor. This migration never +-- fabricates historical Transitions for a Plan 1/2 Situation that predates +-- it: an existing nonterminal or terminal Situation simply gains zero +-- situation_transitions rows and a NULL current_transition_id/0 +-- current_transition_sequence, and remains fully readable. Deciding what to +-- do about that gap (spec: an idempotent "upgrade_reconstruction" due +-- reason so the next fenced reconciliation creates the first truthful +-- Transition) is application-level controller logic for a later task, not +-- a blind migration-time SQL backfill. + +-- ---------------------------------------------------------------------- +-- situation_transitions: the immutable Transition ledger (model.Transition). +-- Exactly one controller-state Transition per material reconciliation, plus +-- one per newly journaled operator artifact (R1). Typed identity/lifecycle/ +-- attention/reason/actor columns for querying and guards; bounded canonical +-- JSON for the Operator contract, the journal render payload, the +-- projection facts, and the evidence-reference list. Rows are inserted +-- once and never touched again — no application ever needs to UPDATE or +-- DELETE a Transition. +-- ---------------------------------------------------------------------- +CREATE TABLE situation_transitions ( + id TEXT NOT NULL PRIMARY KEY CHECK (id <> ''), + situation_id TEXT NOT NULL REFERENCES situations(id), + sequence INTEGER NOT NULL CHECK (sequence >= 1), + input_version INTEGER NOT NULL CHECK (input_version >= 1), + material_fact_hash TEXT NOT NULL CHECK (material_fact_hash <> ''), + assessment_id TEXT REFERENCES situation_assessment_attempts(id), + lifecycle TEXT NOT NULL CHECK (lifecycle IN ('active','recovery_pending','recovered','closed_unknown')), + attention TEXT NOT NULL CHECK (attention IN ('observe','investigate','urgent')), + action_contract_json TEXT NOT NULL CHECK (json_valid(action_contract_json) AND json_type(action_contract_json) = 'object'), + sufficient_reason_id TEXT CHECK (sufficient_reason_id IS NULL OR sufficient_reason_id <> ''), + interruption_priority TEXT CHECK (interruption_priority IS NULL OR interruption_priority IN ('low','medium','high','critical')), + reason TEXT NOT NULL CHECK (reason IN ( + 'first_authoritative_state','material_assessment_changed','attention_changed', + 'operator_contract_changed','investigation_started','investigation_concluded', + 'recovery_observed','recovery_failed','recovered','closed_unknown', + 'recurrence_milestone','triage_state_changed','operator_artifact_recorded' + )), + journal_kind TEXT NOT NULL CHECK (journal_kind IN ( + 'none','publication','investigation_started','investigation_changed', + 'evidence_conclusion','operator_contract_changed','recovery_pending', + 'recovery_refired','recurrence_milestone','recovered','closed_unknown', + 'operator_note','captured_verdict' + )), + journal_json TEXT NOT NULL CHECK (json_valid(journal_json) AND json_type(journal_json) = 'object'), + projection_json TEXT NOT NULL CHECK (json_valid(projection_json) AND json_type(projection_json) = 'object'), + -- R1: the exact situation_input_outbox row this Transition journals, + -- required exactly when reason='operator_artifact_recorded'. + operator_artifact_input_id TEXT REFERENCES situation_input_outbox(id), + evidence_refs_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(evidence_refs_json) AND json_type(evidence_refs_json) = 'array'), + -- Plan 3 may only ever write these three actors; operator_policy is a + -- model.TransitionActor constant reserved for Plan 5 and unconditionally + -- rejected by TransitionActor.Validate, so it is deliberately absent + -- from this CHECK too (a later plan widens this the way 0015 widened + -- llm_health_capabilities' capability enum: rebuild, never edit here). + actor TEXT NOT NULL CHECK (actor IN ('deterministic_controller','llm','attributed_operator')), + drill INTEGER NOT NULL DEFAULT 0 CHECK (drill IN (0, 1)), + created_at TEXT NOT NULL CHECK (created_at <> ''), + CHECK ((reason = 'operator_artifact_recorded') = (operator_artifact_input_id IS NOT NULL)), + UNIQUE (situation_id, sequence) +) STRICT; +CREATE INDEX situation_transitions_operator_artifact_idx ON situation_transitions(operator_artifact_input_id) + WHERE operator_artifact_input_id IS NOT NULL; +CREATE TRIGGER situation_transitions_no_update BEFORE UPDATE ON situation_transitions +BEGIN SELECT RAISE(ABORT, 'situation transition history is immutable'); END; +CREATE TRIGGER situation_transitions_no_delete BEFORE DELETE ON situation_transitions +BEGIN SELECT RAISE(ABORT, 'situation transition history is immutable'); END; +-- Same-Situation authoritative-Assessment guard, mirroring 0015's +-- situations_current_assessment_guard: a Transition may only cite an +-- authoritative Assessment attempt that belongs to its own Situation. +CREATE TRIGGER situation_transitions_assessment_guard BEFORE INSERT ON situation_transitions +WHEN NEW.assessment_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM situation_assessment_attempts a + WHERE a.id = NEW.assessment_id AND a.situation_id = NEW.situation_id AND a.status = 'authoritative' +) +BEGIN SELECT RAISE(ABORT, 'transition assessment_id must reference an authoritative attempt owned by the same situation'); END; + +-- ---------------------------------------------------------------------- +-- situations: the current Transition pointer and its sequence (spec: +-- "current Transition pointer and transition sequence on situations"). A +-- freshly upgraded Plan 1/2 Situation gets NULL/0 (no Transition exists +-- yet); a fresh Plan 3 Situation sets both in the same commit that inserts +-- its first Transition. +-- ---------------------------------------------------------------------- +ALTER TABLE situations ADD COLUMN current_transition_id TEXT REFERENCES situation_transitions(id); +ALTER TABLE situations ADD COLUMN current_transition_sequence INTEGER NOT NULL DEFAULT 0 CHECK (current_transition_sequence >= 0); +-- Same-Situation current-pointer guard, mirroring 0015's +-- situations_current_assessment_guard: current_transition_id must reference +-- a Transition owned by this exact Situation, at the exact sequence this +-- row also claims as current. +CREATE TRIGGER situations_current_transition_guard BEFORE UPDATE OF current_transition_id, current_transition_sequence ON situations +WHEN NEW.current_transition_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM situation_transitions tr + WHERE tr.id = NEW.current_transition_id AND tr.situation_id = NEW.id AND tr.sequence = NEW.current_transition_sequence +) +BEGIN SELECT RAISE(ABORT, 'current_transition_id must reference a transition owned by the same situation with matching sequence'); END; + +-- ---------------------------------------------------------------------- +-- situation_episode_summaries: one CURRENT Episode-summary projection per +-- Situation (model.EpisodeSummary), folded once per Transition in sequence +-- order (Task 4/5's job — this migration only builds the schema and its +-- monotonic guard). Typed identity (situation_id, the table's own PRIMARY +-- KEY), version, and source-transition-sequence fence columns; everything +-- else — title, reasons, evidence conclusion, investigation work, the +-- Operator contract, final outcome, and so on — lives in the bounded +-- canonical summary_json projection. +-- ---------------------------------------------------------------------- +CREATE TABLE situation_episode_summaries ( + situation_id TEXT NOT NULL PRIMARY KEY REFERENCES situations(id), + version INTEGER NOT NULL CHECK (version >= 1), + source_transition_sequence INTEGER NOT NULL CHECK (source_transition_sequence >= 1), + summary_json TEXT NOT NULL CHECK (json_valid(summary_json) AND json_type(summary_json) = 'object'), + updated_at TEXT NOT NULL CHECK (updated_at <> ''), + FOREIGN KEY (situation_id, source_transition_sequence) REFERENCES situation_transitions(situation_id, sequence) +) STRICT; +-- A fold always advances the summary by exactly one version, sourced from a +-- strictly later Transition than the summary it replaces — never a skip, +-- never a replay of an already-summarized (or earlier) Transition. +CREATE TRIGGER situation_episode_summaries_monotonic BEFORE UPDATE ON situation_episode_summaries +WHEN NEW.version != OLD.version + 1 OR NEW.source_transition_sequence <= OLD.source_transition_sequence +BEGIN SELECT RAISE(ABORT, 'episode summary version and source transition sequence must both advance monotonically'); END; + +-- ---------------------------------------------------------------------- +-- situation_transition_stream: the durable stdout-stream outbox, one row +-- per Transition (Task 4/5 inserts it in the same fenced commit that +-- inserts the Transition). status is the STREAM's own closed 3-value set — +-- distinct from both Task 1's NotificationIntent.IntentStatus and this +-- migration's own situation_input_outbox.journal_state — and is never +-- 'claimed': a worker leases a pending row by setting lease_owner/ +-- lease_expires_at (status stays 'pending'), the same shape +-- ClaimDueSituations already uses on situations itself, rather than adding +-- a fourth status value. +-- ---------------------------------------------------------------------- +CREATE TABLE situation_transition_stream ( + id TEXT NOT NULL PRIMARY KEY CHECK (id <> ''), + transition_id TEXT NOT NULL UNIQUE REFERENCES situation_transitions(id), + situation_id TEXT NOT NULL REFERENCES situations(id), + sequence INTEGER NOT NULL CHECK (sequence >= 1), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','delivered','failed')), + lease_owner TEXT, + lease_expires_at TEXT, + claim_token INTEGER NOT NULL DEFAULT 0 CHECK (claim_token >= 0), + attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0), + last_error_class TEXT, + retry_at TEXT, + delivered_at TEXT, + created_at TEXT NOT NULL CHECK (created_at <> ''), + CHECK ((lease_owner IS NULL) = (lease_expires_at IS NULL)), + CHECK ((status = 'delivered') = (delivered_at IS NOT NULL)), + CHECK (status != 'failed' OR retry_at IS NULL) +) STRICT; +CREATE INDEX situation_transition_stream_situation_idx ON situation_transition_stream(situation_id, sequence); +CREATE INDEX situation_transition_stream_claim_idx ON situation_transition_stream(status, lease_expires_at, retry_at, created_at) + WHERE status = 'pending'; +-- The row's identity (which Transition/Situation/sequence/created_at it +-- records) is immutable; a worker may still freely update lease/status/ +-- attempt/retry/delivered_at as delivery proceeds. +CREATE TRIGGER situation_transition_stream_identity_immutable BEFORE UPDATE ON situation_transition_stream +WHEN NEW.transition_id IS NOT OLD.transition_id OR NEW.situation_id IS NOT OLD.situation_id + OR NEW.sequence IS NOT OLD.sequence OR NEW.created_at IS NOT OLD.created_at +BEGIN SELECT RAISE(ABORT, 'situation transition stream identity is immutable'); END; +CREATE TRIGGER situation_transition_stream_no_delete BEFORE DELETE ON situation_transition_stream +BEGIN SELECT RAISE(ABORT, 'situation transition stream history is immutable'); END; + +-- ---------------------------------------------------------------------- +-- situation_input_outbox rebuild (Plan 1's 0014, Plan 2 left it untouched): +-- add exact operator-artifact input provenance and the R1/R2 journaling +-- cursor. STRICT table, so this is create-copy-drop-rename, preserving +-- every existing Plan 1/2 row, index, and CHECK exactly — the new columns +-- are additive and every new CHECK is satisfied by every existing row's +-- default values (journal_state defaults 'not_applicable', which is +-- correct for every kind this table has ever accepted before today). +-- +-- applied_input_version is deliberately NOT paired with status='applied' by +-- any CHECK: only ApplySituationInput's writes going forward stamp it (this +-- migration does not fabricate it for the Plan 1/2 rows it copies forward, +-- so those keep it NULL even though status='applied'). +-- ---------------------------------------------------------------------- +CREATE TABLE situation_input_outbox_new ( + id TEXT NOT NULL PRIMARY KEY CHECK (id <> ''), + idempotency_key TEXT NOT NULL UNIQUE CHECK (idempotency_key <> ''), + incident_id TEXT NOT NULL REFERENCES incidents(id) ON DELETE CASCADE, + delivery_id TEXT REFERENCES alert_deliveries(id), + kind TEXT NOT NULL CHECK (kind IN ( + 'incident_created','membership_changed','incident_ready','finding_persisted', + 'triage_skipped','triage_retry_changed','triage_exhausted','incident_resolved', + 'operator_annotation_recorded','captured_verdict_recorded' + )), + group_key TEXT NOT NULL CHECK (group_key <> ''), + occurred_at TEXT NOT NULL CHECK (occurred_at <> ''), + status TEXT NOT NULL CHECK (status IN ('pending','claimed','applied','failed')), + lease_owner TEXT, + lease_expires_at TEXT, + claim_token INTEGER NOT NULL DEFAULT 0 CHECK (claim_token >= 0), + attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0), + last_error_class TEXT, + retry_at TEXT, + applied_situation_id TEXT REFERENCES situations(id), + applied_at TEXT, + -- The exact situations.input_version this input's application landed + -- (or, for R2's owner-terminal outcome, the terminal owner's own + -- unchanged input_version) stamped. Never fabricated for a pre-Plan-3 + -- row; only ApplySituationInput writes it, going forward. + applied_input_version INTEGER CHECK (applied_input_version IS NULL OR applied_input_version >= 1), + -- The exact durable artifact this row carries — set on exactly the + -- matching kind, never on the other, never on a non-artifact kind. + annotation_id INTEGER REFERENCES incident_annotations(id), + verdict_id INTEGER REFERENCES incident_verdicts(id), + -- R1/R2's journaling cursor: not_applicable for every non-artifact + -- kind; pending from the moment an artifact-kind row is enqueued until + -- either journaled (a Transition consumed it) or owner_terminal (R2: + -- applied to an already-terminal owner, recorded but never journaled). + journal_state TEXT NOT NULL DEFAULT 'not_applicable' + CHECK (journal_state IN ('not_applicable','pending','journaled','owner_terminal')), + journaled_transition_id TEXT REFERENCES situation_transitions(id), + CHECK ((status = 'claimed') = (lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL)), + CHECK ((status = 'applied') = (applied_situation_id IS NOT NULL AND applied_at IS NOT NULL)), + CHECK (status != 'failed' OR retry_at IS NULL), + CHECK ( + (kind = 'operator_annotation_recorded' AND annotation_id IS NOT NULL AND verdict_id IS NULL) OR + (kind = 'captured_verdict_recorded' AND verdict_id IS NOT NULL AND annotation_id IS NULL) OR + (kind NOT IN ('operator_annotation_recorded','captured_verdict_recorded') AND annotation_id IS NULL AND verdict_id IS NULL) + ), + CHECK ((journal_state = 'not_applicable') = (kind NOT IN ('operator_annotation_recorded','captured_verdict_recorded'))), + CHECK ((journal_state = 'journaled') = (journaled_transition_id IS NOT NULL)), + CHECK (journal_state != 'owner_terminal' OR status = 'applied') +) STRICT; + +INSERT INTO situation_input_outbox_new ( + id, idempotency_key, incident_id, delivery_id, kind, group_key, occurred_at, + status, lease_owner, lease_expires_at, claim_token, attempt_count, + last_error_class, retry_at, applied_situation_id, applied_at +) +SELECT id, idempotency_key, incident_id, delivery_id, kind, group_key, occurred_at, + status, lease_owner, lease_expires_at, claim_token, attempt_count, + last_error_class, retry_at, applied_situation_id, applied_at +FROM situation_input_outbox; + +DROP TABLE situation_input_outbox; +ALTER TABLE situation_input_outbox_new RENAME TO situation_input_outbox; + +CREATE INDEX situation_input_outbox_claim_idx ON situation_input_outbox(status, retry_at, occurred_at, id); +-- Supports the controller's R1 read: a Situation's applied, unjournaled +-- artifact rows in (applied_input_version, occurred_at, id) order. +CREATE INDEX situation_input_outbox_pending_journal_idx ON situation_input_outbox(applied_situation_id, journal_state, applied_input_version, occurred_at, id) + WHERE journal_state = 'pending'; diff --git a/internal/store/situation_history_upgrade_test.go b/internal/store/situation_history_upgrade_test.go new file mode 100644 index 0000000..5d5b77b --- /dev/null +++ b/internal/store/situation_history_upgrade_test.go @@ -0,0 +1,793 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "testing" + "time" + + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Step 1: migration 0017 upgrade tests — a populated Plan 2 (migration 16) +// fixture must gain the new STRICT tables and MaxSchemaVersion 17, pass +// PRAGMA foreign_key_check, and acquire zero fabricated Transition history +// for its pre-existing nonterminal/terminal Situations. +// ---------------------------------------------------------------------- + +// seedMigration16HistoryFixture builds a database file shaped like the +// schema immediately before this task's 0017 (every embedded migration +// through version 16 only, so situation_transitions/situation_episode_ +// summaries/situation_transition_stream do not exist yet and +// situation_input_outbox still has its pre-0017 shape) and seeds one +// nonterminal ("active") and one terminal ("closed_unknown") Situation, +// each owning one Incident and one already-"applied" situation_input_outbox +// row — entirely by direct SQL, since the current ApplySituationInput now +// references 0017-only columns (applied_input_version, journal_state) that +// do not exist at this schema version. +func seedMigration16HistoryFixture(t *testing.T, path string) (nonterminalID, terminalID string) { + t.Helper() + ctx := context.Background() + + db, err := sql.Open("sqlite", buildDSN(path)) + if err != nil { + t.Fatalf("open fixture db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ) STRICT; + `); err != nil { + t.Fatalf("create schema_migrations: %v", err) + } + + migrations, err := loadMigrations() + if err != nil { + t.Fatalf("load migrations: %v", err) + } + fixture := &Store{db: db} + for _, m := range migrations { + if m.version > 16 { + continue + } + if err := fixture.applyMigration(ctx, m); err != nil { + t.Fatalf("apply migration %d: %v", m.version, err) + } + } + + now := time.Now().UTC() + nonterminalID = "sit-history-nonterminal" + terminalID = "sit-history-terminal" + + insertOperationalIncident(ctx, t, fixture, "inc-history-nonterminal", "group-history-nonterminal") + if err := insertSituation(ctx, fixture, situationRow{id: nonterminalID, groupKey: "group-history-nonterminal", lifecycle: "active"}); err != nil { + t.Fatalf("insert nonterminal situation: %v", err) + } + if _, err := db.ExecContext(ctx, ` + INSERT INTO situation_incidents (situation_id, incident_id, attached_at) VALUES (?, ?, ?) + `, nonterminalID, "inc-history-nonterminal", canonicalTime(now)); err != nil { + t.Fatalf("attach nonterminal membership: %v", err) + } + if _, err := db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, + status, applied_situation_id, applied_at + ) VALUES ('input-history-nonterminal', 'idem-history-nonterminal', 'inc-history-nonterminal', 'incident_created', 'group-history-nonterminal', ?, 'applied', ?, ?) + `, canonicalTime(now), nonterminalID, canonicalTime(now)); err != nil { + t.Fatalf("insert applied input for nonterminal situation: %v", err) + } + + insertOperationalIncident(ctx, t, fixture, "inc-history-terminal", "group-history-terminal") + term := canonicalTime(now.Add(time.Hour)) + if err := insertSituation(ctx, fixture, situationRow{ + id: terminalID, groupKey: "group-history-terminal", lifecycle: "closed_unknown", + terminalAt: term, terminalReason: "resolution_missing", + }); err != nil { + t.Fatalf("insert terminal situation: %v", err) + } + if _, err := db.ExecContext(ctx, ` + INSERT INTO situation_incidents (situation_id, incident_id, attached_at) VALUES (?, ?, ?) + `, terminalID, "inc-history-terminal", canonicalTime(now)); err != nil { + t.Fatalf("attach terminal membership: %v", err) + } + if _, err := db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, + status, applied_situation_id, applied_at + ) VALUES ('input-history-terminal', 'idem-history-terminal', 'inc-history-terminal', 'incident_created', 'group-history-terminal', ?, 'applied', ?, ?) + `, canonicalTime(now), terminalID, canonicalTime(now)); err != nil { + t.Fatalf("insert applied input for terminal situation: %v", err) + } + + return nonterminalID, terminalID +} + +// TestSituationHistoryUpgrade_CreatesStrictTablesAndBumpsSchemaVersion is +// the brief's literal Step 1 test: opening a migration-16 database with the +// current Open must apply 0017, create its three new STRICT tables, bump +// MaxSchemaVersion to 17, pass PRAGMA foreign_key_check, and leave the +// fixture's pre-existing nonterminal/terminal Situations with zero +// Transitions. +func TestSituationHistoryUpgrade_CreatesStrictTablesAndBumpsSchemaVersion(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migration16-history.db") + nonterminalID, terminalID := seedMigration16HistoryFixture(t, path) + + st, err := Open(ctx, path) + if err != nil { + t.Fatalf("open upgraded store: %v", err) + } + defer func() { _ = st.Close() }() + + var applied int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM schema_migrations WHERE version = 17`).Scan(&applied); err != nil { + t.Fatal(err) + } + if applied != 1 { + t.Fatalf("migration 17 applied count = %d, want 1", applied) + } + + got, err := MaxSchemaVersion() + if err != nil { + t.Fatalf("MaxSchemaVersion: %v", err) + } + if got != 17 { + t.Fatalf("MaxSchemaVersion = %d, want 17", got) + } + + for _, table := range []string{"situation_transitions", "situation_episode_summaries", "situation_transition_stream"} { + var name string + if err := st.DB().QueryRowContext(ctx, `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&name); err != nil { + t.Fatalf("table %s missing after upgrade: %v", table, err) + } + } + + assertNoForeignKeyViolations(ctx, t, st) + + for _, id := range []string{nonterminalID, terminalID} { + var count int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, id).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("situation %s: transitions = %d, want 0 (no fabricated history)", id, count) + } + } +} + +// TestSituationHistoryUpgrade_ExistingSituationsRemainReadableWithZeroHistory +// proves both a pre-Plan-3 nonterminal and a pre-Plan-3 terminal Situation +// stay fully readable after the upgrade, with a NULL/0 current Transition +// pointer — this migration never invents a Transition to fill that gap. +func TestSituationHistoryUpgrade_ExistingSituationsRemainReadableWithZeroHistory(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migration16-history-readable.db") + nonterminalID, terminalID := seedMigration16HistoryFixture(t, path) + + st, err := Open(ctx, path) + if err != nil { + t.Fatalf("open upgraded store: %v", err) + } + defer func() { _ = st.Close() }() + + nonterm, err := st.GetSituation(ctx, nonterminalID) + if err != nil { + t.Fatalf("get nonterminal situation: %v", err) + } + if nonterm.Lifecycle != situationmodel.LifecycleActive { + t.Fatalf("nonterminal lifecycle = %s, want active", nonterm.Lifecycle) + } + + term, err := st.GetSituation(ctx, terminalID) + if err != nil { + t.Fatalf("get terminal situation: %v", err) + } + if term.Lifecycle != situationmodel.LifecycleClosedUnknown { + t.Fatalf("terminal lifecycle = %s, want closed_unknown", term.Lifecycle) + } + + for _, id := range []string{nonterminalID, terminalID} { + var currentTransitionID sql.NullString + var currentTransitionSequence int + if err := st.DB().QueryRowContext(ctx, `SELECT current_transition_id, current_transition_sequence FROM situations WHERE id = ?`, id). + Scan(¤tTransitionID, ¤tTransitionSequence); err != nil { + t.Fatalf("read current transition pointer for %s: %v", id, err) + } + if currentTransitionID.Valid || currentTransitionSequence != 0 { + t.Fatalf("situation %s current transition pointer = (%v,%d), want (NULL,0)", id, currentTransitionID, currentTransitionSequence) + } + } +} + +// ---------------------------------------------------------------------- +// Step 2: direct constraint/trigger tests for situation_transitions, +// situations' new current-Transition pointer, situation_episode_summaries, +// and situation_transition_stream. +// ---------------------------------------------------------------------- + +// transitionRow is a minimal, overridable set of columns for inserting a +// row into situation_transitions directly — schema/constraint tests only; +// the folding logic that builds real Transitions is a later task. +type transitionRow struct { + id string + situationID string + sequence int + inputVersion int + assessmentID any + lifecycle string + attention string + reason string + journalKind string + actor string + operatorArtifactInputID any +} + +func insertTransition(ctx context.Context, s *Store, r transitionRow) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + journal := fmt.Sprintf(`{"headline":"t","occurred_at":"%s"}`, now) + projection := fmt.Sprintf(`{"effective_started_at":"%s","effective_started_at_basis":"source_payload"}`, now) + contract := `{"next_actor":"none","alertint_action":null,"alertint_status":null,"operator_action_required":null,"next_update_at":null,"next_update_on":[],"wait_reason":null}` + inputVersion := r.inputVersion + if inputVersion == 0 { + inputVersion = 1 + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO situation_transitions ( + id, situation_id, sequence, input_version, material_fact_hash, + assessment_id, lifecycle, attention, action_contract_json, + reason, journal_kind, journal_json, projection_json, + operator_artifact_input_id, evidence_refs_json, actor, created_at + ) VALUES (?, ?, ?, ?, 'sha256:mf', ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?, ?) + `, r.id, r.situationID, r.sequence, inputVersion, r.assessmentID, r.lifecycle, r.attention, contract, + r.reason, r.journalKind, journal, projection, r.operatorArtifactInputID, r.actor, now) + return err +} + +// insertAuthoritativeAssessmentAttempt seeds a minimal authoritative +// situation_assessment_attempts row (deterministic_controller derivation, +// no provider call) for the same-Situation Transition-assessment guard +// tests below, and returns its id. sequence must be unique per situationID +// (situation_assessment_attempts' own UNIQUE(situation_id,sequence)). +func insertAuthoritativeAssessmentAttempt(ctx context.Context, t *testing.T, s *Store, id, situationID string, sequence int) string { + t.Helper() + if err := insertAssessmentAttempt(ctx, s, assessmentAttemptRow{ + id: id, situationID: situationID, sequence: sequence, inputVer: 1, workAttempt: 1, + status: "authoritative", derivation: "deterministic_controller", providerStarted: "false", assessmentJSON: "{}", + }); err != nil { + t.Fatalf("seed authoritative assessment attempt %s: %v", id, err) + } + return id +} + +// insertNonAuthoritativeAssessmentAttempt seeds a minimal "stale" +// (non-authoritative) situation_assessment_attempts row and returns its id. +// sequence must be unique per situationID. +func insertNonAuthoritativeAssessmentAttempt(ctx context.Context, t *testing.T, s *Store, id, situationID string, sequence int) string { + t.Helper() + if err := insertAssessmentAttempt(ctx, s, assessmentAttemptRow{ + id: id, situationID: situationID, sequence: sequence, inputVer: 1, workAttempt: 1, + status: "stale", providerStarted: "unknown", + }); err != nil { + t.Fatalf("seed non-authoritative assessment attempt %s: %v", id, err) + } + return id +} + +// TestSituationHistorySchema_TransitionSequenceUniquePerSituation proves the +// (situation_id, sequence) uniqueness constraint. +func TestSituationHistorySchema_TransitionSequenceUniquePerSituation(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-tr-uniq", "group-tr-uniq") + if err := insertSituation(ctx, s, situationRow{id: "sit-tr-uniq", groupKey: "group-tr-uniq", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-1", situationID: "sit-tr-uniq", sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert first transition: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-2", situationID: "sit-tr-uniq", sequence: 1, lifecycle: "active", attention: "observe", + reason: "attention_changed", journalKind: "none", actor: "deterministic_controller", + }); err == nil { + t.Fatal("expected duplicate (situation_id,sequence) to be rejected") + } +} + +// TestSituationHistorySchema_TransitionIsImmutable proves a Transition row +// can never be updated or deleted once inserted. +func TestSituationHistorySchema_TransitionIsImmutable(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-tr-immut", "group-tr-immut") + if err := insertSituation(ctx, s, situationRow{id: "sit-tr-immut", groupKey: "group-tr-immut", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-immut", situationID: "sit-tr-immut", sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition: %v", err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE situation_transitions SET attention = 'urgent' WHERE id = 'tr-immut'`); err == nil { + t.Fatal("expected update of a transition to be rejected") + } + if _, err := s.db.ExecContext(ctx, `DELETE FROM situation_transitions WHERE id = 'tr-immut'`); err == nil { + t.Fatal("expected delete of a transition to be rejected") + } +} + +// TestSituationHistorySchema_TransitionAssessmentGuardRejectsForeignOrNonAuthoritative +// proves the same-Situation authoritative-Assessment guard: a Transition +// may cite only an authoritative attempt owned by its own Situation. +func TestSituationHistorySchema_TransitionAssessmentGuardRejectsForeignOrNonAuthoritative(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-tr-assess", "group-tr-assess") + if err := insertSituation(ctx, s, situationRow{id: "sit-tr-assess", groupKey: "group-tr-assess", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertSituation(ctx, s, situationRow{id: "sit-tr-assess-other", groupKey: "group-tr-assess-other", lifecycle: "active"}); err != nil { + t.Fatalf("insert other situation: %v", err) + } + + nonAuth := insertNonAuthoritativeAssessmentAttempt(ctx, t, s, "attempt-rejected", "sit-tr-assess", 1) + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-bad-attempt", situationID: "sit-tr-assess", sequence: 1, assessmentID: nonAuth, + lifecycle: "active", attention: "observe", reason: "first_authoritative_state", + journalKind: "publication", actor: "deterministic_controller", + }); err == nil { + t.Fatal("expected a non-authoritative assessment_id to be rejected") + } + + authOther := insertAuthoritativeAssessmentAttempt(ctx, t, s, "attempt-other-situation", "sit-tr-assess-other", 1) + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-foreign-attempt", situationID: "sit-tr-assess", sequence: 1, assessmentID: authOther, + lifecycle: "active", attention: "observe", reason: "first_authoritative_state", + journalKind: "publication", actor: "deterministic_controller", + }); err == nil { + t.Fatal("expected a foreign-situation authoritative assessment_id to be rejected") + } + + authSame := insertAuthoritativeAssessmentAttempt(ctx, t, s, "attempt-same-situation", "sit-tr-assess", 2) + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-good-attempt", situationID: "sit-tr-assess", sequence: 1, assessmentID: authSame, + lifecycle: "active", attention: "observe", reason: "first_authoritative_state", + journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("expected a same-situation authoritative assessment_id to be accepted: %v", err) + } +} + +// TestSituationHistorySchema_CurrentTransitionPointerMustBeSameSituation +// proves situations' current-Transition pointer guard: it may reference +// only a Transition owned by that exact Situation, at the matching +// sequence. +func TestSituationHistorySchema_CurrentTransitionPointerMustBeSameSituation(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-ptr", "group-ptr") + if err := insertSituation(ctx, s, situationRow{id: "sit-ptr-a", groupKey: "group-ptr-a", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation a: %v", err) + } + if err := insertSituation(ctx, s, situationRow{id: "sit-ptr-b", groupKey: "group-ptr-b", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation b: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-ptr-a", situationID: "sit-ptr-a", sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition for a: %v", err) + } + + if _, err := s.db.ExecContext(ctx, `UPDATE situations SET current_transition_id = 'tr-ptr-a', current_transition_sequence = 1 WHERE id = 'sit-ptr-b'`); err == nil { + t.Fatal("expected pointing sit-ptr-b at sit-ptr-a's transition to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE situations SET current_transition_id = 'tr-ptr-a', current_transition_sequence = 2 WHERE id = 'sit-ptr-a'`); err == nil { + t.Fatal("expected a mismatched current_transition_sequence to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE situations SET current_transition_id = 'tr-ptr-a', current_transition_sequence = 1 WHERE id = 'sit-ptr-a'`); err != nil { + t.Fatalf("expected a matching same-situation pointer to be accepted: %v", err) + } +} + +// episodeSummaryRow is a minimal, overridable set of columns for inserting +// a row into situation_episode_summaries directly. +type episodeSummaryRow struct { + situationID string + version int + sourceTransitionSequence int +} + +func insertEpisodeSummary(ctx context.Context, s *Store, r episodeSummaryRow) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.ExecContext(ctx, ` + INSERT INTO situation_episode_summaries ( + situation_id, version, source_transition_sequence, summary_json, updated_at + ) VALUES (?, ?, ?, '{"title":"t"}', ?) + `, r.situationID, r.version, r.sourceTransitionSequence, now) + return err +} + +// TestSituationHistorySchema_OneCurrentEpisodeSummaryPerSituation proves the +// PRIMARY KEY(situation_id) invariant: at most one current summary row per +// Situation. +func TestSituationHistorySchema_OneCurrentEpisodeSummaryPerSituation(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-ep-one", "group-ep-one") + if err := insertSituation(ctx, s, situationRow{id: "sit-ep-one", groupKey: "group-ep-one", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-ep-one", situationID: "sit-ep-one", sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition: %v", err) + } + if err := insertEpisodeSummary(ctx, s, episodeSummaryRow{situationID: "sit-ep-one", version: 1, sourceTransitionSequence: 1}); err != nil { + t.Fatalf("insert first episode summary: %v", err) + } + if err := insertEpisodeSummary(ctx, s, episodeSummaryRow{situationID: "sit-ep-one", version: 1, sourceTransitionSequence: 1}); err == nil { + t.Fatal("expected a second situation_episode_summaries row for the same situation to be rejected") + } +} + +// TestSituationHistorySchema_EpisodeSummarySourceSequenceMustReferenceTransition +// proves the composite foreign key: source_transition_sequence must name an +// actual Transition sequence belonging to the same Situation. +func TestSituationHistorySchema_EpisodeSummarySourceSequenceMustReferenceTransition(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-ep-fk", "group-ep-fk") + if err := insertSituation(ctx, s, situationRow{id: "sit-ep-fk", groupKey: "group-ep-fk", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertEpisodeSummary(ctx, s, episodeSummaryRow{situationID: "sit-ep-fk", version: 1, sourceTransitionSequence: 1}); err == nil { + t.Fatal("expected an episode summary with no matching transition to be rejected") + } +} + +// TestSituationHistorySchema_EpisodeSummaryVersionAndSourceSequenceMustAdvance +// proves the monotonic fence: version must advance by exactly 1 and +// source_transition_sequence must strictly increase on every fold. +func TestSituationHistorySchema_EpisodeSummaryVersionAndSourceSequenceMustAdvance(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-ep-mono", "group-ep-mono") + if err := insertSituation(ctx, s, situationRow{id: "sit-ep-mono", groupKey: "group-ep-mono", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + for _, seq := range []int{1, 2} { + if err := insertTransition(ctx, s, transitionRow{ + id: fmt.Sprintf("tr-ep-mono-%d", seq), situationID: "sit-ep-mono", sequence: seq, + lifecycle: "active", attention: "observe", reason: "attention_changed", + journalKind: "none", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition %d: %v", seq, err) + } + } + if err := insertEpisodeSummary(ctx, s, episodeSummaryRow{situationID: "sit-ep-mono", version: 1, sourceTransitionSequence: 1}); err != nil { + t.Fatalf("insert first episode summary: %v", err) + } + + if _, err := s.db.ExecContext(ctx, `UPDATE situation_episode_summaries SET version = 3, source_transition_sequence = 2 WHERE situation_id = 'sit-ep-mono'`); err == nil { + t.Fatal("expected a version skip to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE situation_episode_summaries SET version = 2, source_transition_sequence = 1 WHERE situation_id = 'sit-ep-mono'`); err == nil { + t.Fatal("expected a non-increasing source_transition_sequence to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE situation_episode_summaries SET version = 2, source_transition_sequence = 2 WHERE situation_id = 'sit-ep-mono'`); err != nil { + t.Fatalf("expected a legal fold-forward update to succeed: %v", err) + } +} + +// transitionStreamRow is a minimal set of columns for inserting a row into +// situation_transition_stream directly. +type transitionStreamRow struct { + id string + transitionID string + situationID string + sequence int +} + +func insertTransitionStream(ctx context.Context, s *Store, r transitionStreamRow) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.ExecContext(ctx, ` + INSERT INTO situation_transition_stream ( + id, transition_id, situation_id, sequence, status, created_at + ) VALUES (?, ?, ?, ?, 'pending', ?) + `, r.id, r.transitionID, r.situationID, r.sequence, now) + return err +} + +// TestSituationHistorySchema_TransitionStreamOneRowPerTransition proves the +// UNIQUE(transition_id) constraint. +func TestSituationHistorySchema_TransitionStreamOneRowPerTransition(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-stream-uniq", "group-stream-uniq") + if err := insertSituation(ctx, s, situationRow{id: "sit-stream-uniq", groupKey: "group-stream-uniq", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-stream-uniq", situationID: "sit-stream-uniq", sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition: %v", err) + } + if err := insertTransitionStream(ctx, s, transitionStreamRow{id: "stream-1", transitionID: "tr-stream-uniq", situationID: "sit-stream-uniq", sequence: 1}); err != nil { + t.Fatalf("insert first stream row: %v", err) + } + if err := insertTransitionStream(ctx, s, transitionStreamRow{id: "stream-2", transitionID: "tr-stream-uniq", situationID: "sit-stream-uniq", sequence: 1}); err == nil { + t.Fatal("expected a second stream row for the same transition to be rejected") + } +} + +// TestSituationHistorySchema_TransitionStreamIdentityImmutableButStatusMutable +// proves stream rows reject any change to their identity columns and reject +// delete entirely, while a worker's ordinary lease/status/delivery updates +// succeed. +func TestSituationHistorySchema_TransitionStreamIdentityImmutableButStatusMutable(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-stream-immut", "group-stream-immut") + if err := insertSituation(ctx, s, situationRow{id: "sit-stream-immut", groupKey: "group-stream-immut", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-stream-immut", situationID: "sit-stream-immut", sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition: %v", err) + } + if err := insertTransitionStream(ctx, s, transitionStreamRow{id: "stream-immut", transitionID: "tr-stream-immut", situationID: "sit-stream-immut", sequence: 1}); err != nil { + t.Fatalf("insert stream row: %v", err) + } + + now := time.Now().UTC().Format(time.RFC3339Nano) + if _, err := s.db.ExecContext(ctx, `UPDATE situation_transition_stream SET lease_owner = 'worker-1', lease_expires_at = ? WHERE id = 'stream-immut'`, now); err != nil { + t.Fatalf("expected claiming (lease fields) to be accepted: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE situation_transition_stream SET status = 'delivered', delivered_at = ?, lease_owner = NULL, lease_expires_at = NULL WHERE id = 'stream-immut' + `, now); err != nil { + t.Fatalf("expected delivering to be accepted: %v", err) + } + + if _, err := s.db.ExecContext(ctx, `UPDATE situation_transition_stream SET sequence = 2 WHERE id = 'stream-immut'`); err == nil { + t.Fatal("expected changing sequence identity to be rejected") + } + if _, err := s.db.ExecContext(ctx, `DELETE FROM situation_transition_stream WHERE id = 'stream-immut'`); err == nil { + t.Fatal("expected delete of a stream row to be rejected") + } +} + +// ---------------------------------------------------------------------- +// Step 3: operator-artifact provenance and the R1/R2 journaling cursor on +// the rebuilt situation_input_outbox. +// ---------------------------------------------------------------------- + +// insertAnnotationRow seeds a minimal incident_annotations row and returns +// its rowid, for use as situation_input_outbox.annotation_id. +func insertAnnotationRow(ctx context.Context, s *Store, incidentID string) (int64, error) { + now := time.Now().UTC().Format(time.RFC3339Nano) + res, err := s.db.ExecContext(ctx, ` + INSERT INTO incident_annotations (incident_id, kind, note, created_at) + VALUES (?, 'observation', 'note', ?)`, incidentID, now) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// insertVerdictRow seeds a minimal incident_verdicts row and returns its +// rowid, for use as situation_input_outbox.verdict_id. +func insertVerdictRow(ctx context.Context, s *Store, incidentID string, version int) (int64, error) { + now := time.Now().UTC().Format(time.RFC3339Nano) + res, err := s.db.ExecContext(ctx, ` + INSERT INTO incident_verdicts (incident_id, version, verdict, source, label_confidence, expectation_json, created_at) + VALUES (?, ?, 'confirmation', 'human', 1.0, '{}', ?)`, incidentID, version, now) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// artifactInputRow is a minimal, overridable set of columns for inserting a +// situation_input_outbox row directly, covering both artifact and +// non-artifact kinds for the CHECK tests below. journalState/status default +// to "pending" when left empty. +type artifactInputRow struct { + id string + incidentID string + groupKey string + kind string + annotationID any + verdictID any + journalState string + status string + occurredAt time.Time +} + +func insertArtifactInput(ctx context.Context, s *Store, r artifactInputRow) error { + journalState := r.journalState + if journalState == "" { + journalState = "pending" + } + status := r.status + if status == "" { + status = "pending" + } + occurredAt := r.occurredAt + if occurredAt.IsZero() { + occurredAt = time.Now().UTC() + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, + status, annotation_id, verdict_id, journal_state + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, r.id, "idem:"+r.id, r.incidentID, r.kind, r.groupKey, canonicalTime(occurredAt), status, r.annotationID, r.verdictID, journalState) + return err +} + +// TestOperatorArtifactInputSchema_ReferencePairingMatchesKind proves the +// artifact-reference CHECK: exactly the matching reference on each artifact +// kind, and neither reference on any other kind. +func TestOperatorArtifactInputSchema_ReferencePairingMatchesKind(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-artifact-ref", "group-artifact-ref") + annID, err := insertAnnotationRow(ctx, s, "inc-artifact-ref") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + verdID, err := insertVerdictRow(ctx, s, "inc-artifact-ref", 1) + if err != nil { + t.Fatalf("seed verdict: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-1", incidentID: "inc-artifact-ref", groupKey: "group-artifact-ref", kind: "operator_annotation_recorded"}); err == nil { + t.Fatal("expected annotation kind with no annotation_id to be rejected") + } + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-2", incidentID: "inc-artifact-ref", groupKey: "group-artifact-ref", kind: "operator_annotation_recorded", annotationID: annID, verdictID: verdID}); err == nil { + t.Fatal("expected annotation kind also carrying a verdict_id to be rejected") + } + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-3", incidentID: "inc-artifact-ref", groupKey: "group-artifact-ref", kind: "operator_annotation_recorded", annotationID: annID}); err != nil { + t.Fatalf("expected annotation kind with exactly annotation_id to be accepted: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-4", incidentID: "inc-artifact-ref", groupKey: "group-artifact-ref", kind: "captured_verdict_recorded"}); err == nil { + t.Fatal("expected verdict kind with no verdict_id to be rejected") + } + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-5", incidentID: "inc-artifact-ref", groupKey: "group-artifact-ref", kind: "captured_verdict_recorded", verdictID: verdID}); err != nil { + t.Fatalf("expected verdict kind with exactly verdict_id to be accepted: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-6", incidentID: "inc-artifact-ref", groupKey: "group-artifact-ref", kind: "incident_created", annotationID: annID, journalState: "not_applicable"}); err == nil { + t.Fatal("expected a non-artifact kind carrying annotation_id to be rejected") + } +} + +// TestOperatorArtifactInputSchema_JournalStateMatchesKind proves +// journal_state='not_applicable' iff the kind is not an artifact kind. +func TestOperatorArtifactInputSchema_JournalStateMatchesKind(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-artifact-journal", "group-artifact-journal") + annID, err := insertAnnotationRow(ctx, s, "inc-artifact-journal") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-j1", incidentID: "inc-artifact-journal", groupKey: "group-artifact-journal", kind: "operator_annotation_recorded", annotationID: annID, journalState: "not_applicable"}); err == nil { + t.Fatal("expected an artifact-kind row with journal_state=not_applicable to be rejected") + } + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-j2", incidentID: "inc-artifact-journal", groupKey: "group-artifact-journal", kind: "incident_created", journalState: "pending"}); err == nil { + t.Fatal("expected a non-artifact-kind row with journal_state=pending to be rejected") + } + if err := insertArtifactInput(ctx, s, artifactInputRow{id: "art-j3", incidentID: "inc-artifact-journal", groupKey: "group-artifact-journal", kind: "incident_created", journalState: "not_applicable"}); err != nil { + t.Fatalf("expected a non-artifact-kind row with journal_state=not_applicable to be accepted: %v", err) + } +} + +// TestOperatorArtifactInputSchema_JournaledRequiresTransitionID proves +// journal_state='journaled' iff journaled_transition_id is set, following +// the legal sequence: enqueue pending, a Transition consumes it, then it is +// marked journaled with that Transition's id. +func TestOperatorArtifactInputSchema_JournaledRequiresTransitionID(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-artifact-journaled", "group-artifact-journaled") + annID, err := insertAnnotationRow(ctx, s, "inc-artifact-journaled") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{ + id: "art-journaled-bad", incidentID: "inc-artifact-journaled", groupKey: "group-artifact-journaled", + kind: "operator_annotation_recorded", annotationID: annID, journalState: "journaled", + }); err == nil { + t.Fatal("expected journal_state=journaled with no journaled_transition_id to be rejected") + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{ + id: "art-journaled-ok", incidentID: "inc-artifact-journaled", groupKey: "group-artifact-journaled", + kind: "operator_annotation_recorded", annotationID: annID, journalState: "pending", + }); err != nil { + t.Fatalf("insert pending artifact: %v", err) + } + if err := insertSituation(ctx, s, situationRow{id: "sit-artifact-journaled", groupKey: "group-artifact-journaled-owner", lifecycle: "active"}); err != nil { + t.Fatalf("insert situation: %v", err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: "tr-artifact-journaled", situationID: "sit-artifact-journaled", sequence: 1, lifecycle: "active", + attention: "observe", reason: "operator_artifact_recorded", journalKind: "operator_note", + actor: "attributed_operator", operatorArtifactInputID: "art-journaled-ok", + }); err != nil { + t.Fatalf("insert transition: %v", err) + } + + if _, err := s.db.ExecContext(ctx, ` + UPDATE situation_input_outbox SET journal_state = 'journaled', journaled_transition_id = ? WHERE id = 'art-journaled-ok' + `, "tr-artifact-journaled"); err != nil { + t.Fatalf("expected journal_state=journaled with journaled_transition_id set to be accepted: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{ + id: "art-journaled-2", incidentID: "inc-artifact-journaled", groupKey: "group-artifact-journaled", + kind: "operator_annotation_recorded", annotationID: annID, journalState: "pending", + }); err != nil { + t.Fatalf("insert second pending artifact: %v", err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE situation_input_outbox SET journaled_transition_id = ? WHERE id = 'art-journaled-2'`, "tr-artifact-journaled"); err == nil { + t.Fatal("expected setting journaled_transition_id while journal_state stays pending to be rejected") + } +} + +// TestOperatorArtifactInputSchema_OwnerTerminalRequiresAppliedStatus proves +// journal_state='owner_terminal' is accepted only alongside status='applied'. +func TestOperatorArtifactInputSchema_OwnerTerminalRequiresAppliedStatus(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + insertOperationalIncident(ctx, t, s, "inc-artifact-ownerterm", "group-artifact-ownerterm") + annID, err := insertAnnotationRow(ctx, s, "inc-artifact-ownerterm") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + now := time.Now().UTC() + if err := insertSituation(ctx, s, situationRow{ + id: "sit-artifact-ownerterm", groupKey: "group-artifact-ownerterm-owner", lifecycle: "closed_unknown", + terminalAt: canonicalTime(now), terminalReason: "resolution_missing", + }); err != nil { + t.Fatalf("insert terminal situation: %v", err) + } + + if err := insertArtifactInput(ctx, s, artifactInputRow{ + id: "art-ownerterm-bad", incidentID: "inc-artifact-ownerterm", groupKey: "group-artifact-ownerterm", + kind: "operator_annotation_recorded", annotationID: annID, journalState: "owner_terminal", status: "pending", + }); err == nil { + t.Fatal("expected journal_state=owner_terminal with status!=applied to be rejected") + } + + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, + status, annotation_id, journal_state, applied_situation_id, applied_at + ) VALUES (?, ?, ?, 'operator_annotation_recorded', ?, ?, 'applied', ?, 'owner_terminal', ?, ?) + `, "art-ownerterm-ok", "idem:art-ownerterm-ok", "inc-artifact-ownerterm", "group-artifact-ownerterm", + canonicalTime(now), annID, "sit-artifact-ownerterm", canonicalTime(now)); err != nil { + t.Fatalf("expected journal_state=owner_terminal with status=applied to be accepted: %v", err) + } +} diff --git a/internal/store/situations.go b/internal/store/situations.go index c7311df..3a857b3 100644 --- a/internal/store/situations.go +++ b/internal/store/situations.go @@ -62,11 +62,27 @@ func dueReasonForInputKind(kind string) (situationmodel.DueReason, error) { return situationmodel.DueTriageChanged, nil case "incident_resolved": return situationmodel.DueAlertResolved, nil + case "operator_annotation_recorded", "captured_verdict_recorded": + // R5 (decided): an attributed annotation or a Captured verdict marks + // the Situation due because a durable operator artifact awaits + // journaling — never DueOperatorJudgment, which stays reserved for + // Plan 5's steering catalog. + return situationmodel.DueOperatorArtifactRecorded, nil default: return "", fmt.Errorf("store: unsupported situation input kind %q", kind) } } +// isOperatorArtifactKind reports whether kind is one of the two durable +// operator artifact input kinds (R5): an attributed annotation or a +// Captured verdict. These are the only situation_input_outbox kinds whose +// application follows R1's journaling-cursor discipline and R2's +// owner-terminal handling; every other kind keeps Plan 2's ordinary +// join/create behaviour unconditionally. +func isOperatorArtifactKind(kind string) bool { + return kind == "operator_annotation_recorded" || kind == "captured_verdict_recorded" +} + // ---------------------------------------------------------------------- // situation_input_outbox claim/apply/retry (Task 7) // ---------------------------------------------------------------------- @@ -263,6 +279,17 @@ func readSituationInputTx(ctx context.Context, tx *sql.Tx, id string) (situation // the mapped due reason once, mark applied to the owning Situation. It is // fenced by the SituationClaim's (lease_owner, claim_token) pair and is // idempotent: an input already marked applied is a successful no-op. +// +// R2: for the two operator artifact kinds only, when the Incident's already +// existing owner (situationOwnerForIncidentTx) is terminal +// (recovered/closed_unknown), this does not join — no input_version bump, no +// due reason merged, no lease clear — and instead marks the row applied with +// journal_state='owner_terminal' against that owner. Every other case +// (an active/recovery_pending owner, a fresh group join, or a brand-new +// Situation) follows Plan 2's ordinary join/create path and additionally +// stamps the exact applied_input_version this input landed at, plus +// journal_state='pending' for an artifact kind or 'not_applicable' for +// every other kind (R1). func (s *Store) ApplySituationInput(ctx context.Context, claim SituationClaim) error { if strings.TrimSpace(claim.ID) == "" || strings.TrimSpace(claim.LeaseOwner) == "" || claim.ClaimToken <= 0 { return errors.New("store: apply situation input requires a complete claim") @@ -322,14 +349,31 @@ func (s *Store) ApplySituationInput(ctx context.Context, claim SituationClaim) e } now := time.Now().UTC() - situationID, err := resolveAndApplySituationTx(ctx, tx, row, startAt, basis, receivedAt, dueReason, now) + outcome, err := resolveAndApplySituationTx(ctx, tx, row, startAt, basis, receivedAt, dueReason, now) if err != nil { return err } - if err := attachSituationMembershipTx(ctx, tx, situationID, row.incidentID, now); err != nil { + + if outcome.ownerTerminal { + // R2: the owner resolved at application time is already terminal. + // Record the artifact against it without joining — the terminal + // Episode stays immutable, and no membership attach is needed since + // situationOwnerForIncidentTx already found this exact Incident + // attached to this exact Situation. + if err := markSituationInputAppliedTx(ctx, tx, row.id, claim.LeaseOwner, claim.ClaimToken, outcome.situationID, outcome.inputVersion, "owner_terminal", now); err != nil { + return err + } + return tx.Commit() + } + + if err := attachSituationMembershipTx(ctx, tx, outcome.situationID, row.incidentID, now); err != nil { return err } - if err := markSituationInputAppliedTx(ctx, tx, row.id, claim.LeaseOwner, claim.ClaimToken, situationID, now); err != nil { + journalState := "not_applicable" + if isOperatorArtifactKind(row.kind) { + journalState = "pending" + } + if err := markSituationInputAppliedTx(ctx, tx, row.id, claim.LeaseOwner, claim.ClaimToken, outcome.situationID, outcome.inputVersion, journalState, now); err != nil { return err } return tx.Commit() @@ -378,37 +422,90 @@ func sourceTimesForInputTx(ctx context.Context, tx *sql.Tx, deliveryID *string, return receivedAt, situationmodel.SourceTimeBasisReceiptFallback, receivedAt, nil } +// situationApplyOutcome is resolveAndApplySituationTx's result: which +// Situation the input landed on, the input_version this input's application +// stamped (the resulting post-join/post-create version, or — for R2's +// owner-terminal outcome — the terminal owner's own unchanged current +// version), and whether R2's owner-terminal short-circuit fired. +type situationApplyOutcome struct { + situationID string + inputVersion int + ownerTerminal bool // R2: kind is an artifact kind and the resolved owner is already terminal +} + // resolveAndApplySituationTx implements the owner-selection precedence: an // Incident that already owns a Situation always continues feeding it // (regardless of that Situation's own lifecycle — correlation already // refuses to attach NEW deliveries to an Incident whose owner is terminal, // see terminalSituationOwnerTx, so the only inputs that reach a terminal -// owner here are ones already queued when the owner terminalized — a race -// the future controller must settle explicitly); otherwise the exact -// group's nonterminal Situation joins; otherwise a new active "observe" -// Situation is created, linked via previous_situation_id to the newest -// terminal same-group Situation, if any. It returns the id of the Situation -// the input was applied to. -func resolveAndApplySituationTx(ctx context.Context, tx *sql.Tx, row situationInputRow, startAt time.Time, basis situationmodel.SourceTimeBasis, receivedAt time.Time, dueReason situationmodel.DueReason, now time.Time) (string, error) { - situationID, err := situationOwnerForIncidentTx(ctx, tx, row.incidentID) +// owner here are ones already queued when the owner terminalized); otherwise +// the exact group's nonterminal Situation joins; otherwise a new active +// "observe" Situation is created, linked via previous_situation_id to the +// newest terminal same-group Situation, if any. +// +// R2 settles that terminal-owner race for the two operator artifact kinds +// only: when the Incident's already-existing owner is terminal, this +// short-circuits before ever calling joinSituationTx, returning +// ownerTerminal=true with the owner's own unchanged input_version. Every +// other kind keeps Plan 2's behaviour unconditionally — the future +// controller settling that race for non-artifact kinds is still open, as +// documented above. +func resolveAndApplySituationTx(ctx context.Context, tx *sql.Tx, row situationInputRow, startAt time.Time, basis situationmodel.SourceTimeBasis, receivedAt time.Time, dueReason situationmodel.DueReason, now time.Time) (situationApplyOutcome, error) { + owner, err := situationOwnerForIncidentTx(ctx, tx, row.incidentID) if err != nil { - return "", err + return situationApplyOutcome{}, err } + + if owner != "" && isOperatorArtifactKind(row.kind) { + lifecycle, version, err := situationLifecycleAndVersionTx(ctx, tx, owner) + if err != nil { + return situationApplyOutcome{}, err + } + if lifecycle.Terminal() { + return situationApplyOutcome{situationID: owner, inputVersion: version, ownerTerminal: true}, nil + } + } + + situationID := owner if situationID == "" { situationID, err = nonterminalSituationIDByGroupTx(ctx, tx, row.groupKey) if err != nil { - return "", err + return situationApplyOutcome{}, err } } if situationID != "" { - if err := joinSituationTx(ctx, tx, situationID, startAt, basis, receivedAt, dueReason, row.occurredAt, now); err != nil { - return "", err + version, err := joinSituationTx(ctx, tx, situationID, startAt, basis, receivedAt, dueReason, row.occurredAt, now) + if err != nil { + return situationApplyOutcome{}, err } - return situationID, nil + return situationApplyOutcome{situationID: situationID, inputVersion: version}, nil + } + + newID, err := createSituationTx(ctx, tx, row.groupKey, startAt, basis, receivedAt, dueReason, row.occurredAt, now) + if err != nil { + return situationApplyOutcome{}, err } + return situationApplyOutcome{situationID: newID, inputVersion: 1}, nil +} - return createSituationTx(ctx, tx, row.groupKey, startAt, basis, receivedAt, dueReason, row.occurredAt, now) +// situationLifecycleAndVersionTx reads a Situation's current lifecycle and +// input_version inside an existing transaction, without pulling the full +// row scanSituation would — resolveAndApplySituationTx's R2 owner-terminal +// check needs only these two fields to decide whether an artifact input has +// reached a terminal owner, and if so, at what unchanged input_version to +// stamp. +func situationLifecycleAndVersionTx(ctx context.Context, tx *sql.Tx, id string) (situationmodel.Lifecycle, int, error) { + var lifecycle string + var version int + err := tx.QueryRowContext(ctx, `SELECT lifecycle, input_version FROM situations WHERE id = ?`, id).Scan(&lifecycle, &version) + if errors.Is(err, sql.ErrNoRows) { + return "", 0, ErrNotFound + } + if err != nil { + return "", 0, fmt.Errorf("store: read situation lifecycle: %w", err) + } + return situationmodel.Lifecycle(lifecycle), version, nil } // situationOwnerForIncidentTx returns the Situation id this Incident already @@ -505,11 +602,12 @@ func earlierTime(a, b time.Time) time.Time { // a controller that claimed this Situation before this input's application // holds a lease_owner/claim_token pair that can no longer match once // lease_owner goes NULL here, fencing it out of committing a decision based -// on stale input_version data. -func joinSituationTx(ctx context.Context, tx *sql.Tx, situationID string, startAt time.Time, basis situationmodel.SourceTimeBasis, receivedAt time.Time, dueReason situationmodel.DueReason, occurredAt, now time.Time) error { +// on stale input_version data. Returns the resulting (post-increment) +// input_version. +func joinSituationTx(ctx context.Context, tx *sql.Tx, situationID string, startAt time.Time, basis situationmodel.SourceTimeBasis, receivedAt time.Time, dueReason situationmodel.DueReason, occurredAt, now time.Time) (int, error) { current, err := getSituationTx(ctx, tx, situationID) if err != nil { - return err + return 0, err } newStart := earlierTime(current.EffectiveStartedAt, startAt) @@ -520,7 +618,7 @@ func joinSituationTx(ctx context.Context, tx *sql.Tx, situationID string, startA dueReasonsJSON, err := json.Marshal(newDueReasons) if err != nil { - return fmt.Errorf("store: marshal situation due reasons: %w", err) + return 0, fmt.Errorf("store: marshal situation due reasons: %w", err) } res, err := tx.ExecContext(ctx, ` @@ -534,16 +632,16 @@ func joinSituationTx(ctx context.Context, tx *sql.Tx, situationID string, startA canonicalTime(newStart), string(newBasis), canonicalTime(newFirstReceived), canonicalTime(newNextAssessment), string(dueReasonsJSON), canonicalTime(now), situationID, current.InputVersion) if err != nil { - return fmt.Errorf("store: update situation: %w", err) + return 0, fmt.Errorf("store: update situation: %w", err) } n, err := res.RowsAffected() if err != nil { - return fmt.Errorf("store: count updated situation: %w", err) + return 0, fmt.Errorf("store: count updated situation: %w", err) } if n != 1 { - return ErrSituationVersionConflict + return 0, ErrSituationVersionConflict } - return nil + return current.InputVersion + 1, nil } // createSituationTx inserts a brand-new active "observe" Situation at @@ -600,14 +698,19 @@ func attachSituationMembershipTx(ctx context.Context, tx *sql.Tx, situationID, i // markSituationInputAppliedTx fences the applied transition on the exact // claim this call verified at the top of ApplySituationInput's transaction, -// so nothing else could have moved the lease in between. -func markSituationInputAppliedTx(ctx context.Context, tx *sql.Tx, id, owner string, token int64, situationID string, at time.Time) error { +// so nothing else could have moved the lease in between. appliedInputVersion +// is the exact situations.input_version this input's application landed at +// (join/create's resulting version, or R2's owner-terminal unchanged +// version) and journalState is "not_applicable" for a non-artifact kind, +// "pending" for an artifact kind normally joined, or "owner_terminal" for +// R2's short-circuit. +func markSituationInputAppliedTx(ctx context.Context, tx *sql.Tx, id, owner string, token int64, situationID string, appliedInputVersion int, journalState string, at time.Time) error { res, err := tx.ExecContext(ctx, ` UPDATE situation_input_outbox SET status = 'applied', lease_owner = NULL, lease_expires_at = NULL, retry_at = NULL, - applied_situation_id = ?, applied_at = ? + applied_situation_id = ?, applied_at = ?, applied_input_version = ?, journal_state = ? WHERE id = ? AND status = 'claimed' AND lease_owner = ? AND claim_token = ?`, - situationID, canonicalTime(at), id, owner, token) + situationID, canonicalTime(at), appliedInputVersion, journalState, id, owner, token) if err != nil { return fmt.Errorf("store: mark situation input applied: %w", err) } diff --git a/internal/store/situations_test.go b/internal/store/situations_test.go index 6edfe0a..caac3ed 100644 --- a/internal/store/situations_test.go +++ b/internal/store/situations_test.go @@ -4,6 +4,7 @@ package store import ( "context" + "database/sql" "errors" "fmt" "reflect" @@ -66,6 +67,25 @@ func insertIncidentAndInput(t *testing.T, st *Store, incidentID, inputID, groupK insertIncidentAndInputKind(t, st, incidentID, inputID, groupKey, "incident_created", occurredAt) } +// insertArtifactSituationInput inserts one pending operator-artifact +// situation_input_outbox row (kind operator_annotation_recorded or +// captured_verdict_recorded) for an Incident that already exists, with +// journal_state='pending' from creation — matching what a later task's +// enqueue path will do; this task's own tests exercise only +// ApplySituationInput's R1/R2 handling of an already-pending artifact row, +// never the enqueue path itself. +func insertArtifactSituationInput(t *testing.T, st *Store, incidentID, inputID, groupKey, kind string, annotationID, verdictID any, occurredAt time.Time) { + t.Helper() + if _, err := st.db.ExecContext(context.Background(), ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, + status, annotation_id, verdict_id, journal_state + ) VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, 'pending')`, + inputID, "idem:"+inputID, incidentID, kind, groupKey, canonicalTime(occurredAt), annotationID, verdictID); err != nil { + t.Fatalf("insert artifact situation input %s: %v", inputID, err) + } +} + // insertIncidentAndDeliveryInput inserts a fresh collecting Incident, links // it to an already-accepted delivery's immutable ownership row, and inserts // one pending "membership_changed" situation_input_outbox row referencing @@ -203,14 +223,16 @@ func dueSituationFixture(t *testing.T) (*Store, string, time.Time) { func TestDueReasonForInputKindMapsAllKnownKinds(t *testing.T) { cases := map[string]situationmodel.DueReason{ - "incident_created": situationmodel.DueIncidentCreated, - "membership_changed": situationmodel.DueMembershipChanged, - "incident_ready": situationmodel.DueMembershipChanged, - "finding_persisted": situationmodel.DueNewSymptom, - "triage_skipped": situationmodel.DueTriageChanged, - "triage_retry_changed": situationmodel.DueTriageChanged, - "triage_exhausted": situationmodel.DueTriageChanged, - "incident_resolved": situationmodel.DueAlertResolved, + "incident_created": situationmodel.DueIncidentCreated, + "membership_changed": situationmodel.DueMembershipChanged, + "incident_ready": situationmodel.DueMembershipChanged, + "finding_persisted": situationmodel.DueNewSymptom, + "triage_skipped": situationmodel.DueTriageChanged, + "triage_retry_changed": situationmodel.DueTriageChanged, + "triage_exhausted": situationmodel.DueTriageChanged, + "incident_resolved": situationmodel.DueAlertResolved, + "operator_annotation_recorded": situationmodel.DueOperatorArtifactRecorded, + "captured_verdict_recorded": situationmodel.DueOperatorArtifactRecorded, } for kind, want := range cases { got, err := dueReasonForInputKind(kind) @@ -634,6 +656,251 @@ func TestApplySituationInputClearsControllerLeaseFencingStaleRelease(t *testing. } } +// ---------------------------------------------------------------------- +// Step 5 (R1/R2): ApplySituationInput's handling of the two durable +// operator artifact input kinds. +// ---------------------------------------------------------------------- + +// TestApplySituationInputArtifactAppliedToActiveOwnerMarksPending is the +// R1 "active owner" case: an artifact applied while its owner is +// active/recovery_pending joins normally — input_version bumps, the +// DueOperatorArtifactRecorded reason merges, and the outbox row is stamped +// journal_state='pending' plus the exact applied_input_version it landed at. +func TestApplySituationInputArtifactAppliedToActiveOwnerMarksPending(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 1, 20, 0, 0, 0, time.UTC) + + insertIncidentAndInput(t, st, "inc-artifact-active", "input-artifact-active-seed", "service=artifact-active", now) + seedClaim := claimOneInput(t, st, "w", now) + if err := st.ApplySituationInput(ctx, seedClaim); err != nil { + t.Fatalf("seed apply: %v", err) + } + sits := listSituations(t, st) + if len(sits) != 1 || sits[0].InputVersion != 1 { + t.Fatalf("seed situations = %+v, want exactly 1 at version 1", sits) + } + situationID := sits[0].ID + + annID, err := insertAnnotationRow(ctx, st, "inc-artifact-active") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + insertArtifactSituationInput(t, st, "inc-artifact-active", "input-artifact-active", "service=artifact-active", "operator_annotation_recorded", annID, nil, now.Add(time.Minute)) + artifactClaim := claimOneInput(t, st, "w", now.Add(time.Minute)) + if err := st.ApplySituationInput(ctx, artifactClaim); err != nil { + t.Fatalf("apply artifact input: %v", err) + } + + got := getSituationByID(t, st, situationID) + if got.InputVersion != 2 { + t.Fatalf("input_version = %d, want 2 (artifact bumped it)", got.InputVersion) + } + want := []situationmodel.DueReason{situationmodel.DueIncidentCreated, situationmodel.DueOperatorArtifactRecorded} + if !reflect.DeepEqual(got.DueReasons, want) { + t.Fatalf("due_reasons = %v, want %v", got.DueReasons, want) + } + + var journalState string + var appliedInputVersion sql.NullInt64 + var appliedSituationID sql.NullString + if err := st.db.QueryRowContext(ctx, `SELECT journal_state, applied_input_version, applied_situation_id FROM situation_input_outbox WHERE id = ?`, "input-artifact-active"). + Scan(&journalState, &appliedInputVersion, &appliedSituationID); err != nil { + t.Fatal(err) + } + if journalState != "pending" { + t.Fatalf("journal_state = %q, want pending", journalState) + } + if !appliedInputVersion.Valid || appliedInputVersion.Int64 != 2 { + t.Fatalf("applied_input_version = %v, want 2", appliedInputVersion) + } + if !appliedSituationID.Valid || appliedSituationID.String != situationID { + t.Fatalf("applied_situation_id = %v, want %s", appliedSituationID, situationID) + } +} + +// TestApplySituationInputVerdictArtifactAppliedToActiveOwnerMarksPending +// proves the captured_verdict_recorded kind is treated identically to +// operator_annotation_recorded by isOperatorArtifactKind. +func TestApplySituationInputVerdictArtifactAppliedToActiveOwnerMarksPending(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 2, 0, 0, 0, 0, time.UTC) + + insertIncidentAndInput(t, st, "inc-verdict-active", "input-verdict-active-seed", "service=verdict-active", now) + seedClaim := claimOneInput(t, st, "w", now) + if err := st.ApplySituationInput(ctx, seedClaim); err != nil { + t.Fatalf("seed apply: %v", err) + } + sits := listSituations(t, st) + situationID := sits[0].ID + + verdID, err := insertVerdictRow(ctx, st, "inc-verdict-active", 1) + if err != nil { + t.Fatalf("seed verdict: %v", err) + } + insertArtifactSituationInput(t, st, "inc-verdict-active", "input-verdict-active", "service=verdict-active", "captured_verdict_recorded", nil, verdID, now.Add(time.Minute)) + claim := claimOneInput(t, st, "w", now.Add(time.Minute)) + if err := st.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("apply verdict input: %v", err) + } + + got := getSituationByID(t, st, situationID) + if got.InputVersion != 2 { + t.Fatalf("input_version = %d, want 2", got.InputVersion) + } + var journalState string + if err := st.db.QueryRowContext(ctx, `SELECT journal_state FROM situation_input_outbox WHERE id = 'input-verdict-active'`).Scan(&journalState); err != nil { + t.Fatal(err) + } + if journalState != "pending" { + t.Fatalf("journal_state = %q, want pending", journalState) + } +} + +// TestApplySituationInputArtifactAppliedAfterTerminalOwnerMarksOwnerTerminal +// is the R2 case: an artifact applied after its owner already terminalized +// must not join — input_version and due_reasons_json stay exactly as they +// were, the outbox row is stamped journal_state='owner_terminal', and +// ClaimDueSituations never returns the terminalized owner. +func TestApplySituationInputArtifactAppliedAfterTerminalOwnerMarksOwnerTerminal(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 1, 21, 0, 0, 0, time.UTC) + + insertIncidentAndInput(t, st, "inc-artifact-term", "input-artifact-term-seed", "service=artifact-term", now) + seedClaim := claimOneInput(t, st, "w", now) + if err := st.ApplySituationInput(ctx, seedClaim); err != nil { + t.Fatalf("seed apply: %v", err) + } + sits := listSituations(t, st) + situationID := sits[0].ID + + terminalAt := now.Add(time.Hour) + if _, err := st.db.ExecContext(ctx, ` + UPDATE situations SET lifecycle='closed_unknown', terminal_at=?, terminal_reason='resolution_missing', updated_at=? + WHERE id=?`, canonicalTime(terminalAt), canonicalTime(terminalAt), situationID); err != nil { + t.Fatalf("terminalize fixture situation: %v", err) + } + before := getSituationByID(t, st, situationID) + + annID, err := insertAnnotationRow(ctx, st, "inc-artifact-term") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + insertArtifactSituationInput(t, st, "inc-artifact-term", "input-artifact-term", "service=artifact-term", "operator_annotation_recorded", annID, nil, now.Add(2*time.Hour)) + artifactClaim := claimOneInput(t, st, "w", now.Add(2*time.Hour)) + if err := st.ApplySituationInput(ctx, artifactClaim); err != nil { + t.Fatalf("apply artifact input to terminal owner: %v", err) + } + + after := getSituationByID(t, st, situationID) + if after.InputVersion != before.InputVersion { + t.Fatalf("input_version changed: before %d, after %d, want unchanged", before.InputVersion, after.InputVersion) + } + if !reflect.DeepEqual(after.DueReasons, before.DueReasons) { + t.Fatalf("due_reasons changed: before %v, after %v, want unchanged", before.DueReasons, after.DueReasons) + } + + var journalState string + var appliedSituationID sql.NullString + if err := st.db.QueryRowContext(ctx, `SELECT journal_state, applied_situation_id FROM situation_input_outbox WHERE id = ?`, "input-artifact-term"). + Scan(&journalState, &appliedSituationID); err != nil { + t.Fatal(err) + } + if journalState != "owner_terminal" { + t.Fatalf("journal_state = %q, want owner_terminal", journalState) + } + if !appliedSituationID.Valid || appliedSituationID.String != situationID { + t.Fatalf("applied_situation_id = %v, want %s", appliedSituationID, situationID) + } + + due, err := st.ClaimDueSituations(ctx, "controller-x", now.Add(3*time.Hour), time.Minute, 10) + if err != nil { + t.Fatalf("claim due situations: %v", err) + } + for _, d := range due { + if d.ID == situationID { + t.Fatalf("ClaimDueSituations returned terminalized situation %s", situationID) + } + } +} + +// TestApplySituationInputArtifactActiveOwnerReplayIsNoOp proves idempotent +// replay of the R1 active-owner path: re-applying an already-applied +// artifact claim changes nothing. +func TestApplySituationInputArtifactActiveOwnerReplayIsNoOp(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 1, 22, 0, 0, 0, time.UTC) + + insertIncidentAndInput(t, st, "inc-artifact-replay", "input-artifact-replay-seed", "service=artifact-replay", now) + seedClaim := claimOneInput(t, st, "w", now) + if err := st.ApplySituationInput(ctx, seedClaim); err != nil { + t.Fatalf("seed apply: %v", err) + } + + annID, err := insertAnnotationRow(ctx, st, "inc-artifact-replay") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + insertArtifactSituationInput(t, st, "inc-artifact-replay", "input-artifact-replay", "service=artifact-replay", "operator_annotation_recorded", annID, nil, now.Add(time.Minute)) + claim := claimOneInput(t, st, "w", now.Add(time.Minute)) + if err := st.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("first apply: %v", err) + } + before := listSituations(t, st) + + if err := st.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("replay apply: %v", err) + } + after := listSituations(t, st) + if !reflect.DeepEqual(before, after) { + t.Fatalf("replay changed situations:\nbefore=%+v\nafter=%+v", before, after) + } +} + +// TestApplySituationInputArtifactOwnerTerminalReplayIsNoOp proves idempotent +// replay of the R2 owner-terminal path. +func TestApplySituationInputArtifactOwnerTerminalReplayIsNoOp(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 1, 23, 0, 0, 0, time.UTC) + + insertIncidentAndInput(t, st, "inc-artifact-term-replay", "input-artifact-term-replay-seed", "service=artifact-term-replay", now) + seedClaim := claimOneInput(t, st, "w", now) + if err := st.ApplySituationInput(ctx, seedClaim); err != nil { + t.Fatalf("seed apply: %v", err) + } + sits := listSituations(t, st) + situationID := sits[0].ID + terminalAt := now.Add(time.Hour) + if _, err := st.db.ExecContext(ctx, ` + UPDATE situations SET lifecycle='closed_unknown', terminal_at=?, terminal_reason='resolution_missing', updated_at=? + WHERE id=?`, canonicalTime(terminalAt), canonicalTime(terminalAt), situationID); err != nil { + t.Fatalf("terminalize fixture situation: %v", err) + } + + annID, err := insertAnnotationRow(ctx, st, "inc-artifact-term-replay") + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + insertArtifactSituationInput(t, st, "inc-artifact-term-replay", "input-artifact-term-replay", "service=artifact-term-replay", "operator_annotation_recorded", annID, nil, now.Add(2*time.Hour)) + claim := claimOneInput(t, st, "w", now.Add(2*time.Hour)) + if err := st.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("first apply: %v", err) + } + before := listSituations(t, st) + + if err := st.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("replay apply: %v", err) + } + after := listSituations(t, st) + if !reflect.DeepEqual(before, after) { + t.Fatalf("replay changed situations:\nbefore=%+v\nafter=%+v", before, after) + } +} + // ---------------------------------------------------------------------- // Task 9: read-only Situation views (ListSituations, GetSituation, // GetSituationByHandle, ListSituationIncidents) — the exact surface the MCP diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 7f8b646..51555ed 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -55,6 +55,9 @@ func TestOpen_AppliesEmbeddedMigrations(t *testing.T) { "situation_assessment_attempts": false, "situation_assessment_coverage": false, "incident_triage_attempts": false, + "situation_transitions": false, + "situation_episode_summaries": false, + "situation_transition_stream": false, } for rows.Next() { var name string @@ -440,12 +443,12 @@ func TestMaxSchemaVersion(t *testing.T) { if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - // 0016_incident_triage_controller.sql is the newest migration today. - // Plan 2 owns exactly 0015 and 0016 (the llm_health_capabilities - // widening for "assessment" lives inside 0015); Plan 3 provisionally - // owns 0017/0018, so this number must not move before Plan 2 lands. - if got != 16 { - t.Errorf("MaxSchemaVersion = %d, want 16", got) + // 0017_situation_history.sql is the newest migration today. Plan 2 owns + // 0015/0016; Plan 3 owns exactly 0017/0018 (spec.md "Persistence and + // migration ownership") and this task lands 0017, so the number moves + // from 16 to 17 — 0018 (notification_intents et al.) is a later task. + if got != 17 { + t.Errorf("MaxSchemaVersion = %d, want 17", got) } } From 3ea8b27ec8978b9b7dc3901aa6088605555bca5a Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 00:48:33 +0300 Subject: [PATCH 03/31] feat(store): add durable Situation notification schema Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- .../0018_situation_notifications.sql | 313 +++++ .../situation_notifications_upgrade_test.go | 1124 +++++++++++++++++ internal/store/store_test.go | 15 +- 3 files changed, 1446 insertions(+), 6 deletions(-) create mode 100644 internal/store/migrations/0018_situation_notifications.sql create mode 100644 internal/store/situation_notifications_upgrade_test.go diff --git a/internal/store/migrations/0018_situation_notifications.sql b/internal/store/migrations/0018_situation_notifications.sql new file mode 100644 index 0000000..e2c6887 --- /dev/null +++ b/internal/store/migrations/0018_situation_notifications.sql @@ -0,0 +1,313 @@ +-- SPDX-License-Identifier: FSL-1.1-ALv2 +-- +-- Plan 3 notification-delivery schema: the durable notification_intents +-- ledger (model.NotificationIntent), persisted Situation root Slack +-- coordinates, and the installation-level Delivery-gap tracking tables +-- (slack_delivery_state, slack_delivery_gaps). This migration never +-- fabricates a notification intent or gap for any Plan 1/2/3 Situation that +-- predates it: an upgraded database simply gains the new tables empty, one +-- seeded slack_delivery_state singleton row, and NULL slack_channel/ +-- slack_root_ts on every existing situations row. Deciding when to actually +-- create intents is application-level controller/worker logic for later +-- tasks (Task 4 onward), not a migration-time SQL backfill. + +-- ---------------------------------------------------------------------- +-- situations: persisted Slack root coordinates. Nullable until the +-- Situation's root is durably published (root_sync's first successful +-- delivery); paired so a Situation is never left with a channel and no +-- timestamp or vice versa. Root edits reuse the same coordinates via +-- chat.update, so this migration does not need a history table for them — +-- situation_transition_stream/notification_intents already carry the +-- immutable per-effect delivery history. +-- ---------------------------------------------------------------------- +ALTER TABLE situations ADD COLUMN slack_channel TEXT CHECK (slack_channel IS NULL OR slack_channel <> ''); +ALTER TABLE situations ADD COLUMN slack_root_ts TEXT CHECK (slack_root_ts IS NULL OR slack_root_ts <> ''); +-- ALTER TABLE ADD COLUMN cannot express a cross-column CHECK, so the pairing +-- invariant is a trigger, mirroring 0017's situations_current_transition_guard +-- shape: it fires only when either coordinate is touched, never on the +-- unrelated updates every other situations write already performs. +CREATE TRIGGER situations_slack_root_pairing_guard BEFORE UPDATE OF slack_channel, slack_root_ts ON situations +WHEN (NEW.slack_channel IS NULL) != (NEW.slack_root_ts IS NULL) +BEGIN SELECT RAISE(ABORT, 'situation slack_channel and slack_root_ts must be set or unset together'); END; + +-- ---------------------------------------------------------------------- +-- notification_intents: one durable, fenced Slack delivery obligation per +-- model.NotificationIntent, created inside the authoritative controller +-- commit (Task 5) and claimed/delivered by the notification worker +-- (Task 6/7). Rows are inserted once and never deleted, but — unlike +-- situation_transitions — they are NOT wholesale immutable: status, claim, +-- retry, supersession, and delivery columns mutate across the intent's own +-- lifecycle (claim -> retry -> deliver, or claim -> block -> redrive, or +-- pending -> superseded). Only the columns that name WHAT this intent is +-- (effect class, subject references, poke/priority, the deadline it +-- renders, its client message id, and its creation time) are immutable +-- once inserted. +-- +-- Effect classes (model.EffectClass) are exactly root_sync, thread_append, +-- broadcast_handoff, and installation_gap_recovery. installation_gap_recovery +-- is the one class with no Situation/Transition/summary reference — it +-- references a slack_delivery_gaps generation instead. The other three +-- reference a Situation and the Transition that created their content; +-- root_sync additionally references the Episode-summary version it renders +-- (thread_append/broadcast_handoff render only their own Transition's +-- stored journal data, never the newest Situation, so they carry no summary +-- reference — spec.md "Notification intent contract"). +-- ---------------------------------------------------------------------- +CREATE TABLE notification_intents ( + id TEXT NOT NULL PRIMARY KEY CHECK (id <> ''), + idempotency_key TEXT NOT NULL UNIQUE CHECK (idempotency_key <> ''), + effect_class TEXT NOT NULL CHECK (effect_class IN ( + 'root_sync','thread_append','broadcast_handoff','installation_gap_recovery' + )), + situation_id TEXT REFERENCES situations(id), + transition_id TEXT REFERENCES situation_transitions(id), + -- Denormalized alongside transition_id (same shape as + -- situation_transition_stream's own situation_id+sequence columns) so + -- the thread/broadcast and root-pending partial unique indexes below + -- don't need a join. The identity-guard trigger keeps it truthful. + transition_sequence INTEGER CHECK (transition_sequence IS NULL OR transition_sequence >= 1), + -- root_sync only (R4): the Episode-summary version this root renders. + summary_version INTEGER CHECK (summary_version IS NULL OR summary_version >= 1), + -- installation_gap_recovery only. + gap_generation TEXT REFERENCES slack_delivery_gaps(id), + -- Denormalized derived fact: true for exactly thread_append and + -- broadcast_handoff (spec.md "a root must be durably delivered before + -- any reply is claimable"). root_sync creates or edits the root itself + -- and installation_gap_recovery has no Situation root to depend on, so + -- both are false. Enforced by the requires_root CHECK below rather than + -- left to worker logic, so a future effect class cannot silently ship + -- without deciding this. + requires_root INTEGER NOT NULL CHECK (requires_root IN (0, 1)), + main_channel_poke INTEGER NOT NULL CHECK (main_channel_poke IN (0, 1)), + -- Set if and only if main_channel_poke is true: the InterruptionPriority + -- this poke candidate was evaluated against the notify.slack.min_severity + -- floor with (spec.md "Publication authority and Interruption priority"). + -- A non-poke effect (a plain root edit, an ordinary journal reply, or the + -- installation-level gap-recovery notice) carries no Interruption + -- priority of its own. + interruption_priority TEXT CHECK (interruption_priority IS NULL OR interruption_priority IN ('low','medium','high','critical')), + -- root_sync only (R4): the committed promise this root renders, captured + -- at commit time from the committed Operator contract — never from the + -- Episode summary, and never on any other effect class. + contract_deadline_at TEXT, + client_message_id TEXT NOT NULL CHECK (client_message_id <> ''), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ( + 'pending','delivered','blocked_configuration','failed', + 'withheld_by_operator_slack_floor','superseded' + )), + claim_owner TEXT, + claim_token INTEGER NOT NULL DEFAULT 0 CHECK (claim_token >= 0), + lease_expires_at TEXT, + -- Attempt count is preserved across every status transition, including + -- into and out of blocked_configuration: no column here expresses a + -- maximum-attempt terminal outcome, since only 'failed' (an invalid + -- durable intent or non-recoverable programming/data error) is + -- terminal-by-attempts, and even that is explicitly operator-redriveable + -- (spec.md "Required fields and states"). + attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0), + last_error_class TEXT, + retry_at TEXT, + supersession_reason TEXT, + replacement_intent_id TEXT REFERENCES notification_intents(id), + -- The delivery-mode a completed Deliver() call reported (worker-internal + -- NotificationDelivery.DeliveredAs), not present on the wire contract + -- Task 1 closed: root | thread | broadcast | delayed_thread | system. + -- delayed_thread covers both an ordinary-delay backlog entry and a + -- broadcast_handoff revalidated stale and downgraded to a non-broadcast, + -- no-longer-current entry (spec.md "Recovery replay"). + delivered_as TEXT CHECK (delivered_as IS NULL OR delivered_as IN ('root','thread','broadcast','delayed_thread','system')), + channel TEXT, + message_ts TEXT, + created_at TEXT NOT NULL CHECK (created_at <> ''), + delivered_at TEXT, + + -- Effect-class-specific reference shape (model.NotificationIntent.Validate + -- mirrored at the store layer): installation_gap_recovery carries no + -- Situation/Transition/summary reference and requires a gap generation; + -- every other class requires a Situation/Transition reference and + -- forbids a gap generation. + CHECK ( + (effect_class = 'installation_gap_recovery' + AND situation_id IS NULL AND transition_id IS NULL AND transition_sequence IS NULL + AND gap_generation IS NOT NULL) + OR + (effect_class != 'installation_gap_recovery' + AND situation_id IS NOT NULL AND transition_id IS NOT NULL AND transition_sequence IS NOT NULL + AND gap_generation IS NULL) + ), + -- summary_version is set if and only if effect_class = 'root_sync'. + CHECK ((summary_version IS NOT NULL) = (effect_class = 'root_sync')), + -- contract_deadline_at is nullable even on root_sync (a terminal root has + -- no pending promise), but non-NULL only ever appears on root_sync (R4). + CHECK (contract_deadline_at IS NULL OR effect_class = 'root_sync'), + -- requires_root is the deterministic function of effect_class described + -- above, never an independently-set flag. + CHECK (requires_root = (effect_class IN ('thread_append','broadcast_handoff'))), + -- Main-poke/priority equivalence: a candidate main-channel poke always + -- carries the Interruption priority it was floor-evaluated against + -- (including a poke immediately withheld by that floor), and nothing + -- that isn't a poke candidate carries one. + CHECK ((main_channel_poke = 1) = (interruption_priority IS NOT NULL)), + -- Claim-owner/lease pairing, and claiming only ever moves a pending + -- intent's lease deadline (spec.md "Claims retain pending"). + CHECK ((claim_owner IS NULL) = (lease_expires_at IS NULL)), + CHECK (claim_owner IS NULL OR status = 'pending'), + -- failed is never auto-retried (mirrors 0017's situation_transition_stream + -- and situation_input_outbox: status != 'failed' OR retry_at IS NULL). + CHECK (status != 'failed' OR retry_at IS NULL), + -- Delivered coordinates/time consistency: all four or none. + CHECK ((status = 'delivered') = (delivered_at IS NOT NULL AND channel IS NOT NULL AND message_ts IS NOT NULL AND delivered_as IS NOT NULL)), + -- Only a pending root projection may ever become superseded — an + -- immutable thread_append/broadcast_handoff journal entry never is + -- (spec.md "Older root projections ... may become superseded ... + -- Distinct episodes and material journal entries never become + -- superseded"), and every superseded row records why and by what. + CHECK (status != 'superseded' OR effect_class = 'root_sync'), + CHECK ((status = 'superseded') = (supersession_reason IS NOT NULL)), + CHECK ((status = 'superseded') = (replacement_intent_id IS NOT NULL)), + CHECK (replacement_intent_id IS NULL OR replacement_intent_id != id) +) STRICT; + +-- Same-Situation Transition guard, mirroring 0014/0017's current-pointer and +-- Assessment guards: an intent's transition_id must belong to the same +-- Situation this row claims, at the exact sequence this row also claims. +-- INSERT-only: transition_id/transition_sequence/situation_id are part of +-- this row's immutable identity (enforced below), so no legal UPDATE can +-- ever change them. +CREATE TRIGGER notification_intents_transition_guard BEFORE INSERT ON notification_intents +WHEN NEW.transition_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM situation_transitions tr + WHERE tr.id = NEW.transition_id AND tr.situation_id = NEW.situation_id AND tr.sequence = NEW.transition_sequence +) +BEGIN SELECT RAISE(ABORT, 'notification intent transition_id must reference a transition owned by the same situation with matching sequence'); END; + +-- Identity is immutable; status/claim/retry/supersession/delivery columns +-- are the intent's mutable lifecycle and may be freely updated. +CREATE TRIGGER notification_intents_identity_immutable BEFORE UPDATE ON notification_intents +WHEN NEW.idempotency_key IS NOT OLD.idempotency_key OR NEW.effect_class IS NOT OLD.effect_class + OR NEW.situation_id IS NOT OLD.situation_id OR NEW.transition_id IS NOT OLD.transition_id + OR NEW.transition_sequence IS NOT OLD.transition_sequence OR NEW.summary_version IS NOT OLD.summary_version + OR NEW.gap_generation IS NOT OLD.gap_generation OR NEW.requires_root IS NOT OLD.requires_root + OR NEW.main_channel_poke IS NOT OLD.main_channel_poke OR NEW.interruption_priority IS NOT OLD.interruption_priority + OR NEW.contract_deadline_at IS NOT OLD.contract_deadline_at OR NEW.client_message_id IS NOT OLD.client_message_id + OR NEW.created_at IS NOT OLD.created_at +BEGIN SELECT RAISE(ABORT, 'notification intent identity is immutable'); END; +-- No DELETE, ever — this is durable delivery history, not a queue that +-- forgets. Unlike situation_transitions there is deliberately no +-- "_no_update" trigger here: the lifecycle columns above are the whole +-- point of this table's mutability. +CREATE TRIGGER notification_intents_no_delete BEFORE DELETE ON notification_intents +BEGIN SELECT RAISE(ABORT, 'notification intent history is immutable'); END; +-- Only a currently-pending root_sync may become superseded — never one +-- already delivered/blocked/failed/withheld (those are already resolved +-- outcomes, not live candidates a newer root_sync coalesces away), and +-- never a second hop through an already-superseded row. +CREATE TRIGGER notification_intents_supersede_from_pending_only BEFORE UPDATE OF status ON notification_intents +WHEN NEW.status = 'superseded' AND OLD.status != 'pending' +BEGIN SELECT RAISE(ABORT, 'only a pending root_sync intent may become superseded'); END; + +-- At most one pending, unsuperseded root_sync per Situation: a root_sync +-- refresh (R4) or a newly warranted root edit must supersede any existing +-- pending root_sync before (or in the same commit as) inserting its +-- replacement, rather than ever letting two compete for the same root. +CREATE UNIQUE INDEX notification_intents_root_sync_pending_idx ON notification_intents(situation_id) + WHERE effect_class = 'root_sync' AND status = 'pending'; +-- thread_append and broadcast_handoff are immutable historical effects: at +-- most one per (Situation, Transition sequence, effect class), for all +-- time, not just while pending. root_sync is deliberately excluded — a +-- root_sync refresh (R4) reuses its authority Transition's sequence on +-- purpose, so its own idempotency key (which also folds in summary_version +-- and contract_deadline_at) is what keeps it unique, not this index. +CREATE UNIQUE INDEX notification_intents_thread_broadcast_uniq_idx ON notification_intents(situation_id, transition_sequence, effect_class) + WHERE effect_class IN ('thread_append','broadcast_handoff'); +-- Due-claim ordering: the worker's claim query scans exactly this shape, +-- mirroring situation_transition_stream_claim_idx. +CREATE INDEX notification_intents_claim_idx ON notification_intents(status, lease_expires_at, retry_at, created_at) + WHERE status = 'pending'; +-- Root-dependency lookups: "has this Situation's root already been +-- delivered" (thread_append/broadcast_handoff claimability) and "what is +-- the current root_sync's status" both scan this shape. +CREATE INDEX notification_intents_root_dependency_idx ON notification_intents(situation_id, status) + WHERE effect_class = 'root_sync'; +-- Situation ordering: a Situation's intents in Transition-sequence order, +-- the shape spec.md's ordering rules (root before reply, sequence order, +-- handoff-edit before broadcast) read against. +CREATE INDEX notification_intents_situation_order_idx ON notification_intents(situation_id, transition_sequence, id) + WHERE situation_id IS NOT NULL; +-- Gap-recovery replay: locate the one installation_gap_recovery intent for +-- a given generation (to confirm it delivered before backlog replay +-- starts). +CREATE INDEX notification_intents_gap_generation_idx ON notification_intents(gap_generation) + WHERE gap_generation IS NOT NULL; + +-- ---------------------------------------------------------------------- +-- slack_delivery_gaps: one row per installation-level Delivery-gap +-- generation (spec.md "Gap lifecycle" / "Recovery replay"). open while +-- continuous Slack failures persist, replaying once a readiness check +-- succeeds and the recovery notice + backlog delivery is underway, complete +-- once the backlog finishes. Rows are inserted once (open) and mutate +-- forward through this lifecycle; never deleted. +-- ---------------------------------------------------------------------- +CREATE TABLE slack_delivery_gaps ( + id TEXT NOT NULL PRIMARY KEY CHECK (id <> ''), + status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open','replaying','complete')), + opened_at TEXT NOT NULL CHECK (opened_at <> ''), + affected_situation_count INTEGER NOT NULL DEFAULT 0 CHECK (affected_situation_count >= 0), + delayed_effect_count INTEGER NOT NULL DEFAULT 0 CHECK (delayed_effect_count >= 0), + -- Set once the gap starts replaying: the claimable installation_gap_recovery + -- intent that must deliver before the affected-Situation backlog does. + recovery_notice_intent_id TEXT REFERENCES notification_intents(id), + recovered_at TEXT, + completed_at TEXT, + CHECK (status != 'open' OR (recovered_at IS NULL AND completed_at IS NULL AND recovery_notice_intent_id IS NULL)), + CHECK (status != 'replaying' OR (recovered_at IS NOT NULL AND completed_at IS NULL)), + CHECK (status != 'complete' OR (recovered_at IS NOT NULL AND completed_at IS NOT NULL)) +) STRICT; +-- id/opened_at are this generation's immutable identity; status, the +-- counts, the recovery-notice reference, and the recovered/completed +-- instants are its mutable lifecycle. +CREATE TRIGGER slack_delivery_gaps_identity_immutable BEFORE UPDATE ON slack_delivery_gaps +WHEN NEW.id IS NOT OLD.id OR NEW.opened_at IS NOT OLD.opened_at +BEGIN SELECT RAISE(ABORT, 'slack delivery gap identity is immutable'); END; +CREATE TRIGGER slack_delivery_gaps_no_delete BEFORE DELETE ON slack_delivery_gaps +BEGIN SELECT RAISE(ABORT, 'slack delivery gap history is immutable'); END; + +-- ---------------------------------------------------------------------- +-- slack_delivery_state: the one aggregate Slack-dependency-health row, +-- exactly the fixed-id singleton idiom 0012_llm_health.sql already +-- established for the LLM-dependency aggregate. Seeded here so the worker +-- (Task 6/7) only ever UPDATEs it. +-- ---------------------------------------------------------------------- +CREATE TABLE slack_delivery_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + first_failure_at TEXT, + last_success_at TEXT, + -- The gap generation currently open or replaying, or NULL when Slack + -- delivery is healthy or the last gap has fully completed. Guarded below + -- so this can never point at an already-complete generation. + open_gap_generation TEXT REFERENCES slack_delivery_gaps(id), + -- Bumped when startup detects corrected Slack configuration (spec.md + -- "blocked_configuration ... Startup with corrected configuration + -- increments a durable configuration generation"); moves affected + -- intents back to pending independently of the gap-generation lifecycle + -- above. + configuration_generation INTEGER NOT NULL DEFAULT 0 CHECK (configuration_generation >= 0), + -- The last time a bounded retry WARN was emitted, so the worker can + -- honor the "bounded retry WARNs at dependency health cadence" pacing + -- without re-deriving it from notification_intents on every tick. + last_warning_at TEXT, + updated_at TEXT NOT NULL CHECK (updated_at <> '') +) STRICT; + +INSERT INTO slack_delivery_state (id, configuration_generation, updated_at) +VALUES (1, 0, strftime('%Y-%m-%dT%H:%M:%fZ', 'now')); + +-- open_gap_generation may only ever name a gap still open or replaying — +-- never one already complete, and never a generation that doesn't exist. +CREATE TRIGGER slack_delivery_state_open_gap_guard BEFORE UPDATE OF open_gap_generation ON slack_delivery_state +WHEN NEW.open_gap_generation IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM slack_delivery_gaps g WHERE g.id = NEW.open_gap_generation AND g.status IN ('open','replaying') +) +BEGIN SELECT RAISE(ABORT, 'slack delivery state open_gap_generation must reference an open or replaying gap'); END; +CREATE TRIGGER slack_delivery_state_no_delete BEFORE DELETE ON slack_delivery_state +BEGIN SELECT RAISE(ABORT, 'slack delivery state is a singleton and may not be deleted'); END; diff --git a/internal/store/situation_notifications_upgrade_test.go b/internal/store/situation_notifications_upgrade_test.go new file mode 100644 index 0000000..277b2bd --- /dev/null +++ b/internal/store/situation_notifications_upgrade_test.go @@ -0,0 +1,1124 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" +) + +// ---------------------------------------------------------------------- +// Step 5: migration 0018 upgrade tests — a populated migration-17 fixture +// (Task 2's history schema already applied) must gain the new STRICT +// tables and MaxSchemaVersion 18, pass PRAGMA foreign_key_check, gain +// NULL slack_channel/slack_root_ts on every existing situations row, and +// acquire zero fabricated notification_intents/slack_delivery_gaps rows. +// ---------------------------------------------------------------------- + +// seedMigration17NotificationsFixture builds a database file shaped like +// the schema immediately before this task's 0018 (every embedded +// migration through version 17 only, so notification_intents/ +// slack_delivery_state/slack_delivery_gaps and situations.slack_channel/ +// slack_root_ts do not exist yet) and seeds three Situations covering the +// states the brief's Step 5 names: one nonterminal with pending controller +// work (a due, unclaimed reconciliation), one carrying blocked/retry state +// (a claimed-and-failed lease with a scheduled retry), and one terminal — +// entirely by direct SQL, since this fixture predates 0018's columns. +func seedMigration17NotificationsFixture(t *testing.T, path string) (pendingID, retryID, terminalID string) { + t.Helper() + ctx := context.Background() + + db, err := sql.Open("sqlite", buildDSN(path)) + if err != nil { + t.Fatalf("open fixture db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ) STRICT; + `); err != nil { + t.Fatalf("create schema_migrations: %v", err) + } + + migrations, err := loadMigrations() + if err != nil { + t.Fatalf("load migrations: %v", err) + } + fixture := &Store{db: db} + for _, m := range migrations { + if m.version > 17 { + continue + } + if err := fixture.applyMigration(ctx, m); err != nil { + t.Fatalf("apply migration %d: %v", m.version, err) + } + } + + now := time.Now().UTC() + pendingID = "sit-notif-pending" + retryID = "sit-notif-retry" + terminalID = "sit-notif-terminal" + + insertOperationalIncident(ctx, t, fixture, "inc-notif-pending", "group-notif-pending") + if err := insertSituation(ctx, fixture, situationRow{id: pendingID, groupKey: "group-notif-pending", lifecycle: "active"}); err != nil { + t.Fatalf("insert pending situation: %v", err) + } + + insertOperationalIncident(ctx, t, fixture, "inc-notif-retry", "group-notif-retry") + if err := insertSituation(ctx, fixture, situationRow{id: retryID, groupKey: "group-notif-retry", lifecycle: "active"}); err != nil { + t.Fatalf("insert retry situation: %v", err) + } + // Blocked/retry state: a claimed lease that failed and is now scheduled + // for retry, the same shape ClaimDueSituations/its failure path leaves + // behind — proves the upgrade preserves this in-flight controller state + // verbatim. + if _, err := db.ExecContext(ctx, ` + UPDATE situations SET last_error_class = 'llm_timeout', retry_at = ?, attempt_count = 2 WHERE id = ? + `, canonicalTime(now.Add(5*time.Minute)), retryID); err != nil { + t.Fatalf("seed retry state: %v", err) + } + + insertOperationalIncident(ctx, t, fixture, "inc-notif-terminal", "group-notif-terminal") + term := canonicalTime(now.Add(time.Hour)) + if err := insertSituation(ctx, fixture, situationRow{ + id: terminalID, groupKey: "group-notif-terminal", lifecycle: "closed_unknown", + terminalAt: term, terminalReason: "resolution_missing", + }); err != nil { + t.Fatalf("insert terminal situation: %v", err) + } + + return pendingID, retryID, terminalID +} + +// TestSituationNotificationsUpgrade_CreatesStrictTablesAndBumpsSchemaVersion +// is the brief's literal Step 5 test: opening a migration-17 database with +// the current Open must apply 0018, create its new STRICT tables, bump +// MaxSchemaVersion to 18, pass PRAGMA foreign_key_check, seed exactly one +// slack_delivery_state row, and fabricate zero notification_intents or +// slack_delivery_gaps rows. +func TestSituationNotificationsUpgrade_CreatesStrictTablesAndBumpsSchemaVersion(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migration17-notifications.db") + seedMigration17NotificationsFixture(t, path) + + st, err := Open(ctx, path) + if err != nil { + t.Fatalf("open upgraded store: %v", err) + } + defer func() { _ = st.Close() }() + + var applied int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM schema_migrations WHERE version = 18`).Scan(&applied); err != nil { + t.Fatal(err) + } + if applied != 1 { + t.Fatalf("migration 18 applied count = %d, want 1", applied) + } + + got, err := MaxSchemaVersion() + if err != nil { + t.Fatalf("MaxSchemaVersion: %v", err) + } + if got != 18 { + t.Fatalf("MaxSchemaVersion = %d, want 18", got) + } + + for _, table := range []string{"notification_intents", "slack_delivery_gaps", "slack_delivery_state"} { + var name string + if err := st.DB().QueryRowContext(ctx, `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&name); err != nil { + t.Fatalf("table %s missing after upgrade: %v", table, err) + } + } + + assertNoForeignKeyViolations(ctx, t, st) + + var intents, gaps, states int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM notification_intents`).Scan(&intents); err != nil { + t.Fatal(err) + } + if intents != 0 { + t.Fatalf("notification_intents count = %d, want 0 (migration alone fabricates nothing)", intents) + } + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM slack_delivery_gaps`).Scan(&gaps); err != nil { + t.Fatal(err) + } + if gaps != 0 { + t.Fatalf("slack_delivery_gaps count = %d, want 0 (migration alone fabricates nothing)", gaps) + } + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM slack_delivery_state`).Scan(&states); err != nil { + t.Fatal(err) + } + if states != 1 { + t.Fatalf("slack_delivery_state count = %d, want exactly 1 seeded singleton", states) + } + + var configGen int + var openGap sql.NullString + if err := st.DB().QueryRowContext(ctx, `SELECT configuration_generation, open_gap_generation FROM slack_delivery_state WHERE id = 1`). + Scan(&configGen, &openGap); err != nil { + t.Fatalf("read seeded slack_delivery_state: %v", err) + } + if configGen != 0 || openGap.Valid { + t.Fatalf("seeded slack_delivery_state = (configuration_generation=%d, open_gap_generation=%v), want (0, NULL)", configGen, openGap) + } +} + +// TestSituationNotificationsUpgrade_ExistingSituationsGainNoSlackRoot +// proves every pre-0018 Situation — pending controller work, blocked/retry +// state, and terminal — remains fully readable after the upgrade, with +// NULL slack_channel/slack_root_ts: this migration never invents a +// published root, and it preserves in-flight retry/attempt state exactly. +func TestSituationNotificationsUpgrade_ExistingSituationsGainNoSlackRoot(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migration17-notifications-readable.db") + pendingID, retryID, terminalID := seedMigration17NotificationsFixture(t, path) + + st, err := Open(ctx, path) + if err != nil { + t.Fatalf("open upgraded store: %v", err) + } + defer func() { _ = st.Close() }() + + for _, id := range []string{pendingID, retryID, terminalID} { + var channel, rootTS sql.NullString + if err := st.DB().QueryRowContext(ctx, `SELECT slack_channel, slack_root_ts FROM situations WHERE id = ?`, id). + Scan(&channel, &rootTS); err != nil { + t.Fatalf("read slack root coordinates for %s: %v", id, err) + } + if channel.Valid || rootTS.Valid { + t.Fatalf("situation %s slack root = (%v,%v), want (NULL,NULL)", id, channel, rootTS) + } + } + + var lastErrorClass sql.NullString + var attemptCount int + if err := st.DB().QueryRowContext(ctx, `SELECT last_error_class, attempt_count FROM situations WHERE id = ?`, retryID). + Scan(&lastErrorClass, &attemptCount); err != nil { + t.Fatalf("read retry situation: %v", err) + } + if !lastErrorClass.Valid || lastErrorClass.String != "llm_timeout" || attemptCount != 2 { + t.Fatalf("retry situation state = (%v,%d), want (llm_timeout,2) preserved across upgrade", lastErrorClass, attemptCount) + } + + nonterm, err := st.GetSituation(ctx, pendingID) + if err != nil { + t.Fatalf("get pending situation: %v", err) + } + if nonterm.Lifecycle != "active" { + t.Fatalf("pending situation lifecycle = %s, want active", nonterm.Lifecycle) + } + + term, err := st.GetSituation(ctx, terminalID) + if err != nil { + t.Fatalf("get terminal situation: %v", err) + } + if term.Lifecycle != "closed_unknown" { + t.Fatalf("terminal situation lifecycle = %s, want closed_unknown", term.Lifecycle) + } +} + +// ---------------------------------------------------------------------- +// Step 1/2: direct constraint tests for notification_intents. +// ---------------------------------------------------------------------- + +// notificationIntentRow is a minimal, overridable set of columns for +// inserting a row into notification_intents directly — schema/constraint +// tests only; the planning/commit logic that builds real intents is a +// later task. Every field defaults to a legal root_sync shape unless +// overridden, since that is the most field-heavy effect class. +type notificationIntentRow struct { + id string + idempotencyKey string + effectClass string + situationID any + transitionID any + transitionSequence any + summaryVersion any + gapGeneration any + requiresRoot int + mainChannelPoke int + interruptionPriority any + contractDeadlineAt any + clientMessageID string + status string + claimOwner any + leaseExpiresAt any + supersessionReason any + replacementIntentID any + deliveredAs any + channel any + messageTS any + deliveredAt any +} + +func insertNotificationIntent(ctx context.Context, s *Store, r notificationIntentRow) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + status := r.status + if status == "" { + status = "pending" + } + clientMessageID := r.clientMessageID + if clientMessageID == "" { + clientMessageID = "cmid-" + r.id + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + summary_version, gap_generation, requires_root, main_channel_poke, interruption_priority, + contract_deadline_at, client_message_id, status, claim_owner, lease_expires_at, + supersession_reason, replacement_intent_id, delivered_as, channel, message_ts, + created_at, delivered_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, r.id, r.idempotencyKey, r.effectClass, r.situationID, r.transitionID, r.transitionSequence, + r.summaryVersion, r.gapGeneration, r.requiresRoot, r.mainChannelPoke, r.interruptionPriority, + r.contractDeadlineAt, clientMessageID, status, r.claimOwner, r.leaseExpiresAt, + r.supersessionReason, r.replacementIntentID, r.deliveredAs, r.channel, r.messageTS, + now, r.deliveredAt) + return err +} + +// legalRootSyncIntent returns a notificationIntentRow shaped as a legal +// pending root_sync referencing the given situation/transition, for tests +// that only need one valid anchor row to mutate or conflict against. +func legalRootSyncIntent(id, situationID, transitionID string, sequence, summaryVersion int) notificationIntentRow { + return notificationIntentRow{ + id: id, idempotencyKey: "idem-" + id, effectClass: "root_sync", + situationID: situationID, transitionID: transitionID, transitionSequence: sequence, + summaryVersion: summaryVersion, requiresRoot: 0, mainChannelPoke: 1, interruptionPriority: "high", + } +} + +func seedNotificationIntentFixture(ctx context.Context, t *testing.T, s *Store, situationID, groupKey, transitionID string) { + t.Helper() + insertOperationalIncident(ctx, t, s, "inc-"+situationID, groupKey) + if err := insertSituation(ctx, s, situationRow{id: situationID, groupKey: groupKey, lifecycle: "active"}); err != nil { + t.Fatalf("insert situation %s: %v", situationID, err) + } + if err := insertTransition(ctx, s, transitionRow{ + id: transitionID, situationID: situationID, sequence: 1, lifecycle: "active", attention: "observe", + reason: "first_authoritative_state", journalKind: "publication", actor: "deterministic_controller", + }); err != nil { + t.Fatalf("insert transition %s: %v", transitionID, err) + } +} + +// TestSituationNotificationSchema_ClosedEffectClassAndStatus proves the +// closed effect_class and status enums. +func TestSituationNotificationSchema_ClosedEffectClassAndStatus(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-enum", "group-ni-enum", "tr-ni-enum") + + bad := legalRootSyncIntent("ni-enum-bad-class", "sit-ni-enum", "tr-ni-enum", 1, 1) + bad.effectClass = "bogus_class" + if err := insertNotificationIntent(ctx, s, bad); err == nil { + t.Fatal("expected an unknown effect_class to be rejected") + } + + badStatus := legalRootSyncIntent("ni-enum-bad-status", "sit-ni-enum", "tr-ni-enum", 1, 1) + badStatus.status = "bogus_status" + if err := insertNotificationIntent(ctx, s, badStatus); err == nil { + t.Fatal("expected an unknown status to be rejected") + } + + good := legalRootSyncIntent("ni-enum-good", "sit-ni-enum", "tr-ni-enum", 1, 1) + if err := insertNotificationIntent(ctx, s, good); err != nil { + t.Fatalf("expected a legal root_sync intent to be accepted: %v", err) + } +} + +// TestSituationNotificationSchema_ClosedDeliveryMode proves delivered_as is +// a closed enum: root | thread | broadcast | delayed_thread | system. +func TestSituationNotificationSchema_ClosedDeliveryMode(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-mode", "group-ni-mode", "tr-ni-mode") + + bad := legalRootSyncIntent("ni-mode-bad", "sit-ni-mode", "tr-ni-mode", 1, 1) + bad.status = "delivered" + bad.deliveredAs = "bogus_mode" + bad.channel, bad.messageTS = "C1", "111.222" + bad.deliveredAt = time.Now().UTC().Format(time.RFC3339Nano) + if err := insertNotificationIntent(ctx, s, bad); err == nil { + t.Fatal("expected an unknown delivered_as value to be rejected") + } + + for _, mode := range []string{"root", "thread", "broadcast", "delayed_thread", "system"} { + row := legalRootSyncIntent("ni-mode-"+mode, "sit-ni-mode", "tr-ni-mode", 1, 1) + row.status = "delivered" + row.deliveredAs = mode + row.channel, row.messageTS = "C1", "111.222" + row.deliveredAt = time.Now().UTC().Format(time.RFC3339Nano) + if err := insertNotificationIntent(ctx, s, row); err != nil { + t.Fatalf("expected delivered_as=%s to be accepted: %v", mode, err) + } + } +} + +// TestSituationNotificationSchema_UTCInstants proves created_at is required +// and non-empty (the store convention of UTC RFC3339Nano text timestamps — +// a blank value is the one shape a CHECK can reject directly). +func TestSituationNotificationSchema_UTCInstants(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-utc", "group-ni-utc", "tr-ni-utc") + + row := legalRootSyncIntent("ni-utc-bad", "sit-ni-utc", "tr-ni-utc", 1, 1) + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + summary_version, requires_root, main_channel_poke, interruption_priority, + client_message_id, status, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '') + `, row.id, row.idempotencyKey, row.effectClass, row.situationID, row.transitionID, row.transitionSequence, + row.summaryVersion, row.requiresRoot, row.mainChannelPoke, row.interruptionPriority, + "cmid-ni-utc-bad", "pending"); err == nil { + t.Fatal("expected an empty created_at to be rejected") + } +} + +// TestSituationNotificationSchema_StableIdempotencyAndClientIDs proves +// idempotency_key is unique and client_message_id is required. +func TestSituationNotificationSchema_StableIdempotencyAndClientIDs(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-idem", "group-ni-idem", "tr-ni-idem") + + first := legalRootSyncIntent("ni-idem-1", "sit-ni-idem", "tr-ni-idem", 1, 1) + if err := insertNotificationIntent(ctx, s, first); err != nil { + t.Fatalf("insert first intent: %v", err) + } + dup := legalRootSyncIntent("ni-idem-2", "sit-ni-idem", "tr-ni-idem", 1, 1) + dup.idempotencyKey = first.idempotencyKey + if err := insertNotificationIntent(ctx, s, dup); err == nil { + t.Fatal("expected a duplicate idempotency_key to be rejected") + } + + // A blank client_message_id is rejected. Uses thread_append (not + // root_sync) so the only thing under test is the client_message_id + // CHECK, not the separate one-pending-root_sync-per-situation index — + // insertNotificationIntent's own helper can't exercise this: it + // substitutes a synthesized id whenever clientMessageID is left blank, + // so this needs a direct INSERT with a literal empty string. + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + requires_root, main_channel_poke, client_message_id, status, created_at + ) VALUES (?, ?, 'thread_append', ?, ?, ?, ?, ?, '', 'pending', ?) + `, "ni-idem-blank", "idem-ni-idem-blank", "sit-ni-idem", "tr-ni-idem", 1, 1, 0, + time.Now().UTC().Format(time.RFC3339Nano)); err == nil { + t.Fatal("expected an empty client_message_id to be rejected") + } +} + +// TestSituationNotificationSchema_ReferenceShapeBySituationAndGapClass +// proves the effect-class reference shape: the three Situation effects +// require Situation/Transition/sequence and forbid gap_generation; +// installation_gap_recovery requires gap_generation and forbids the rest. +func TestSituationNotificationSchema_ReferenceShapeBySituationAndGapClass(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-ref", "group-ni-ref", "tr-ni-ref") + if err := insertGapGeneration(ctx, s, "gap-ni-ref", "open"); err != nil { + t.Fatalf("seed gap generation: %v", err) + } + + // A Situation effect missing situation_id/transition_id is rejected. + missing := legalRootSyncIntent("ni-ref-missing", "sit-ni-ref", "tr-ni-ref", 1, 1) + missing.situationID = nil + if err := insertNotificationIntent(ctx, s, missing); err == nil { + t.Fatal("expected root_sync with no situation_id to be rejected") + } + + // A Situation effect also carrying gap_generation is rejected. + both := legalRootSyncIntent("ni-ref-both", "sit-ni-ref", "tr-ni-ref", 1, 1) + both.gapGeneration = "gap-ni-ref" + if err := insertNotificationIntent(ctx, s, both); err == nil { + t.Fatal("expected root_sync also carrying gap_generation to be rejected") + } + + // installation_gap_recovery with a situation_id is rejected. + gapWithSituation := notificationIntentRow{ + id: "ni-ref-gap-bad", idempotencyKey: "idem-ni-ref-gap-bad", effectClass: "installation_gap_recovery", + situationID: "sit-ni-ref", gapGeneration: "gap-ni-ref", requiresRoot: 0, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, gapWithSituation); err == nil { + t.Fatal("expected installation_gap_recovery with a situation_id to be rejected") + } + + // installation_gap_recovery with no gap_generation is rejected. + gapMissing := notificationIntentRow{ + id: "ni-ref-gap-missing", idempotencyKey: "idem-ni-ref-gap-missing", effectClass: "installation_gap_recovery", + requiresRoot: 0, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, gapMissing); err == nil { + t.Fatal("expected installation_gap_recovery with no gap_generation to be rejected") + } + + // A legal installation_gap_recovery intent is accepted. + gapGood := notificationIntentRow{ + id: "ni-ref-gap-good", idempotencyKey: "idem-ni-ref-gap-good", effectClass: "installation_gap_recovery", + gapGeneration: "gap-ni-ref", requiresRoot: 0, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, gapGood); err != nil { + t.Fatalf("expected a legal installation_gap_recovery intent to be accepted: %v", err) + } +} + +// TestSituationNotificationSchema_SummaryVersionOnlyOnRootSync proves +// summary_version is set if and only if effect_class = 'root_sync'. +func TestSituationNotificationSchema_SummaryVersionOnlyOnRootSync(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-sv", "group-ni-sv", "tr-ni-sv") + + rootNoVersion := legalRootSyncIntent("ni-sv-root-missing", "sit-ni-sv", "tr-ni-sv", 1, 1) + rootNoVersion.summaryVersion = nil + if err := insertNotificationIntent(ctx, s, rootNoVersion); err == nil { + t.Fatal("expected root_sync with no summary_version to be rejected") + } + + threadWithVersion := notificationIntentRow{ + id: "ni-sv-thread-bad", idempotencyKey: "idem-ni-sv-thread-bad", effectClass: "thread_append", + situationID: "sit-ni-sv", transitionID: "tr-ni-sv", transitionSequence: 1, summaryVersion: 1, + requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, threadWithVersion); err == nil { + t.Fatal("expected thread_append carrying summary_version to be rejected") + } + + threadGood := notificationIntentRow{ + id: "ni-sv-thread-good", idempotencyKey: "idem-ni-sv-thread-good", effectClass: "thread_append", + situationID: "sit-ni-sv", transitionID: "tr-ni-sv", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, threadGood); err != nil { + t.Fatalf("expected thread_append with no summary_version to be accepted: %v", err) + } +} + +// TestSituationNotificationSchema_ContractDeadlineOnlyOnRootSync (R4) +// proves contract_deadline_at is nullable but non-NULL only on root_sync. +func TestSituationNotificationSchema_ContractDeadlineOnlyOnRootSync(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-deadline", "group-ni-deadline", "tr-ni-deadline") + + now := time.Now().UTC().Format(time.RFC3339Nano) + + threadWithDeadline := notificationIntentRow{ + id: "ni-deadline-thread-bad", idempotencyKey: "idem-ni-deadline-thread-bad", effectClass: "thread_append", + situationID: "sit-ni-deadline", transitionID: "tr-ni-deadline", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 0, contractDeadlineAt: now, + } + if err := insertNotificationIntent(ctx, s, threadWithDeadline); err == nil { + t.Fatal("expected thread_append carrying contract_deadline_at to be rejected") + } + + rootNoDeadline := legalRootSyncIntent("ni-deadline-root-nil", "sit-ni-deadline", "tr-ni-deadline", 1, 1) + if err := insertNotificationIntent(ctx, s, rootNoDeadline); err != nil { + t.Fatalf("expected root_sync with no contract_deadline_at (terminal root) to be accepted: %v", err) + } + // Free the situation's one-pending-root_sync slot before inserting a + // second root_sync below. + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'delivered', channel = 'C1', message_ts = '111.222', + delivered_as = 'root', delivered_at = ? WHERE id = ? + `, now, rootNoDeadline.id); err != nil { + t.Fatalf("mark rootNoDeadline delivered: %v", err) + } + + rootWithDeadline := legalRootSyncIntent("ni-deadline-root-set", "sit-ni-deadline", "tr-ni-deadline", 1, 2) + rootWithDeadline.contractDeadlineAt = now + if err := insertNotificationIntent(ctx, s, rootWithDeadline); err != nil { + t.Fatalf("expected root_sync carrying contract_deadline_at to be accepted: %v", err) + } +} + +// TestSituationNotificationSchema_MainPokePriorityEquivalence proves a +// candidate main-channel poke always carries an Interruption priority, and +// nothing else does. +func TestSituationNotificationSchema_MainPokePriorityEquivalence(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-poke", "group-ni-poke", "tr-ni-poke") + + pokeNoPriority := legalRootSyncIntent("ni-poke-missing", "sit-ni-poke", "tr-ni-poke", 1, 1) + pokeNoPriority.mainChannelPoke = 1 + pokeNoPriority.interruptionPriority = nil + if err := insertNotificationIntent(ctx, s, pokeNoPriority); err == nil { + t.Fatal("expected main_channel_poke=true with no interruption_priority to be rejected") + } + + priorityNoPoke := legalRootSyncIntent("ni-poke-extra", "sit-ni-poke", "tr-ni-poke", 1, 1) + priorityNoPoke.mainChannelPoke = 0 + priorityNoPoke.interruptionPriority = "high" + if err := insertNotificationIntent(ctx, s, priorityNoPoke); err == nil { + t.Fatal("expected main_channel_poke=false carrying interruption_priority to be rejected") + } + + good := legalRootSyncIntent("ni-poke-good", "sit-ni-poke", "tr-ni-poke", 1, 1) + good.mainChannelPoke = 0 + good.interruptionPriority = nil + if err := insertNotificationIntent(ctx, s, good); err != nil { + t.Fatalf("expected a non-poke with no priority to be accepted: %v", err) + } +} + +// TestSituationNotificationSchema_RootRequirementMatchesEffectClass proves +// requires_root is exactly true for thread_append/broadcast_handoff and +// false otherwise. +func TestSituationNotificationSchema_RootRequirementMatchesEffectClass(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-root", "group-ni-root", "tr-ni-root") + if err := insertGapGeneration(ctx, s, "gap-ni-root", "open"); err != nil { + t.Fatalf("seed gap generation: %v", err) + } + + rootSyncWrong := legalRootSyncIntent("ni-root-rs-bad", "sit-ni-root", "tr-ni-root", 1, 1) + rootSyncWrong.requiresRoot = 1 + if err := insertNotificationIntent(ctx, s, rootSyncWrong); err == nil { + t.Fatal("expected root_sync with requires_root=true to be rejected") + } + + threadWrong := notificationIntentRow{ + id: "ni-root-thread-bad", idempotencyKey: "idem-ni-root-thread-bad", effectClass: "thread_append", + situationID: "sit-ni-root", transitionID: "tr-ni-root", transitionSequence: 1, + requiresRoot: 0, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, threadWrong); err == nil { + t.Fatal("expected thread_append with requires_root=false to be rejected") + } + + threadGood := notificationIntentRow{ + id: "ni-root-thread-good", idempotencyKey: "idem-ni-root-thread-good", effectClass: "thread_append", + situationID: "sit-ni-root", transitionID: "tr-ni-root", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, threadGood); err != nil { + t.Fatalf("expected thread_append with requires_root=true to be accepted: %v", err) + } + + gapWrong := notificationIntentRow{ + id: "ni-root-gap-bad", idempotencyKey: "idem-ni-root-gap-bad", effectClass: "installation_gap_recovery", + gapGeneration: "gap-ni-root", requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, gapWrong); err == nil { + t.Fatal("expected installation_gap_recovery with requires_root=true to be rejected") + } +} + +// TestSituationNotificationSchema_ClaimOwnerTokenLeaseConsistency proves +// claim_owner/lease_expires_at are paired and claiming is only legal while +// status='pending'. +func TestSituationNotificationSchema_ClaimOwnerTokenLeaseConsistency(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-claim", "group-ni-claim", "tr-ni-claim") + + unpaired := legalRootSyncIntent("ni-claim-unpaired", "sit-ni-claim", "tr-ni-claim", 1, 1) + unpaired.claimOwner = "worker-1" + unpaired.leaseExpiresAt = nil + if err := insertNotificationIntent(ctx, s, unpaired); err == nil { + t.Fatal("expected claim_owner with no lease_expires_at to be rejected") + } + + claimedButDelivered := legalRootSyncIntent("ni-claim-delivered", "sit-ni-claim", "tr-ni-claim", 1, 1) + claimedButDelivered.claimOwner = "worker-1" + claimedButDelivered.leaseExpiresAt = time.Now().UTC().Format(time.RFC3339Nano) + claimedButDelivered.status = "delivered" + claimedButDelivered.deliveredAs = "root" + claimedButDelivered.channel = "C1" + claimedButDelivered.messageTS = "111.222" + claimedButDelivered.deliveredAt = time.Now().UTC().Format(time.RFC3339Nano) + if err := insertNotificationIntent(ctx, s, claimedButDelivered); err == nil { + t.Fatal("expected a claim_owner set while status != pending to be rejected") + } + + good := legalRootSyncIntent("ni-claim-good", "sit-ni-claim", "tr-ni-claim", 1, 1) + good.claimOwner = "worker-1" + good.leaseExpiresAt = time.Now().UTC().Format(time.RFC3339Nano) + if err := insertNotificationIntent(ctx, s, good); err != nil { + t.Fatalf("expected a pending claimed intent to be accepted: %v", err) + } + + var claimToken int + if err := s.db.QueryRowContext(ctx, `SELECT claim_token FROM notification_intents WHERE id = ?`, good.id).Scan(&claimToken); err != nil { + t.Fatalf("read default claim_token: %v", err) + } + if claimToken != 0 { + t.Fatalf("default claim_token = %d, want 0", claimToken) + } + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET claim_token = -1 WHERE id = ?`, good.id); err == nil { + t.Fatal("expected a negative claim_token to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET claim_token = 3 WHERE id = ?`, good.id); err != nil { + t.Fatalf("expected bumping claim_token to succeed: %v", err) + } +} + +// TestSituationNotificationSchema_AttemptCountPreservedAcrossConfigurationBlock +// proves attempt_count survives a pending -> blocked_configuration -> +// pending round trip untouched: no schema field resets or caps it, and +// blocked_configuration is a durable, indefinitely-held state rather than +// an attempt-exhaustion outcome. +func TestSituationNotificationSchema_AttemptCountPreservedAcrossConfigurationBlock(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-attempts", "group-ni-attempts", "tr-ni-attempts") + + row := legalRootSyncIntent("ni-attempts", "sit-ni-attempts", "tr-ni-attempts", 1, 1) + if err := insertNotificationIntent(ctx, s, row); err != nil { + t.Fatalf("insert intent: %v", err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET attempt_count = 5 WHERE id = ?`, row.id); err != nil { + t.Fatalf("bump attempt_count: %v", err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET status = 'blocked_configuration' WHERE id = ?`, row.id); err != nil { + t.Fatalf("block on configuration: %v", err) + } + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET status = 'pending' WHERE id = ?`, row.id); err != nil { + t.Fatalf("redrive back to pending: %v", err) + } + + var attemptCount int + if err := s.db.QueryRowContext(ctx, `SELECT attempt_count FROM notification_intents WHERE id = ?`, row.id).Scan(&attemptCount); err != nil { + t.Fatalf("read attempt_count: %v", err) + } + if attemptCount != 5 { + t.Fatalf("attempt_count after blocked_configuration round trip = %d, want 5 (preserved, not reset)", attemptCount) + } +} + +// TestSituationNotificationSchema_DeliveredCoordinatesAndTimeConsistency +// proves delivered_at/channel/message_ts/delivered_as are set together iff +// status='delivered'. +func TestSituationNotificationSchema_DeliveredCoordinatesAndTimeConsistency(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-delivered", "group-ni-delivered", "tr-ni-delivered") + + now := time.Now().UTC().Format(time.RFC3339Nano) + + partial := legalRootSyncIntent("ni-delivered-partial", "sit-ni-delivered", "tr-ni-delivered", 1, 1) + partial.status = "delivered" + partial.channel = "C1" + partial.messageTS = "111.222" + // deliveredAt and deliveredAs left unset — must be rejected. + if err := insertNotificationIntent(ctx, s, partial); err == nil { + t.Fatal("expected status=delivered with incomplete delivery coordinates to be rejected") + } + + claimedNotDelivered := legalRootSyncIntent("ni-delivered-early", "sit-ni-delivered", "tr-ni-delivered", 1, 1) + claimedNotDelivered.channel = "C1" + claimedNotDelivered.messageTS = "111.222" + claimedNotDelivered.deliveredAt = now + claimedNotDelivered.deliveredAs = "root" + // status stays 'pending' while delivery coordinates are already set — + // must be rejected. + if err := insertNotificationIntent(ctx, s, claimedNotDelivered); err == nil { + t.Fatal("expected delivery coordinates set while status != delivered to be rejected") + } + + good := legalRootSyncIntent("ni-delivered-good", "sit-ni-delivered", "tr-ni-delivered", 1, 1) + good.status = "delivered" + good.channel = "C1" + good.messageTS = "111.222" + good.deliveredAt = now + good.deliveredAs = "root" + if err := insertNotificationIntent(ctx, s, good); err != nil { + t.Fatalf("expected a fully delivered intent to be accepted: %v", err) + } +} + +// ---------------------------------------------------------------------- +// Step 2: indexes and history safety. +// ---------------------------------------------------------------------- + +// TestSituationNotificationSchema_OnlyOnePendingUnsupersededRootSync proves +// the partial unique index: at most one pending root_sync per Situation. +func TestSituationNotificationSchema_OnlyOnePendingUnsupersededRootSync(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-onepending", "group-ni-onepending", "tr-ni-onepending") + + first := legalRootSyncIntent("ni-onepending-1", "sit-ni-onepending", "tr-ni-onepending", 1, 1) + if err := insertNotificationIntent(ctx, s, first); err != nil { + t.Fatalf("insert first pending root_sync: %v", err) + } + + second := legalRootSyncIntent("ni-onepending-2", "sit-ni-onepending", "tr-ni-onepending", 1, 2) + if err := insertNotificationIntent(ctx, s, second); err == nil { + t.Fatal("expected a second pending root_sync for the same situation to be rejected") + } + + // placeholder stands in for the real replacement intent that a live + // controller commit would create in the same transaction (Task 5); this + // schema-only test just needs any legal, already-existing row to satisfy + // replacement_intent_id's FK and NOT-NULL-on-superseded requirements. + placeholder := notificationIntentRow{ + id: "ni-onepending-placeholder", idempotencyKey: "idem-ni-onepending-placeholder", effectClass: "thread_append", + situationID: "sit-ni-onepending", transitionID: "tr-ni-onepending", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, placeholder); err != nil { + t.Fatalf("insert placeholder: %v", err) + } + + // Superseding the first frees the slot for a new pending root_sync. + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'superseded', supersession_reason = 'newer_root', replacement_intent_id = ? + WHERE id = 'ni-onepending-1' + `, placeholder.id); err != nil { + t.Fatalf("supersede first root_sync: %v", err) + } + if err := insertNotificationIntent(ctx, s, second); err != nil { + t.Fatalf("expected a new pending root_sync to be accepted once the prior one is superseded: %v", err) + } +} + +// TestSituationNotificationSchema_ThreadAndBroadcastUniquePerTransition +// proves the partial unique index on (situation_id, transition_sequence, +// effect_class) for thread_append/broadcast_handoff, and that root_sync is +// exempt (a root_sync refresh may legally reuse the same sequence). +func TestSituationNotificationSchema_ThreadAndBroadcastUniquePerTransition(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-tb-uniq", "group-ni-tb-uniq", "tr-ni-tb-uniq") + + thread1 := notificationIntentRow{ + id: "ni-tb-uniq-thread-1", idempotencyKey: "idem-ni-tb-uniq-thread-1", effectClass: "thread_append", + situationID: "sit-ni-tb-uniq", transitionID: "tr-ni-tb-uniq", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, thread1); err != nil { + t.Fatalf("insert first thread_append: %v", err) + } + thread2 := thread1 + thread2.id, thread2.idempotencyKey = "ni-tb-uniq-thread-2", "idem-ni-tb-uniq-thread-2" + if err := insertNotificationIntent(ctx, s, thread2); err == nil { + t.Fatal("expected a second thread_append for the same (situation,sequence) to be rejected") + } + + broadcast1 := notificationIntentRow{ + id: "ni-tb-uniq-broadcast-1", idempotencyKey: "idem-ni-tb-uniq-broadcast-1", effectClass: "broadcast_handoff", + situationID: "sit-ni-tb-uniq", transitionID: "tr-ni-tb-uniq", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 1, interruptionPriority: "high", + } + if err := insertNotificationIntent(ctx, s, broadcast1); err != nil { + t.Fatalf("expected broadcast_handoff to coexist with thread_append at the same sequence: %v", err) + } + + // root_sync is exempt: two root_sync rows at the same sequence (an + // initial post plus an R4 deadline refresh) are legal at the schema + // level as long as only one stays pending (proven separately above), so + // mark the first delivered before inserting the second at the same + // sequence. + root1 := legalRootSyncIntent("ni-tb-uniq-root-1", "sit-ni-tb-uniq", "tr-ni-tb-uniq", 1, 1) + if err := insertNotificationIntent(ctx, s, root1); err != nil { + t.Fatalf("insert first root_sync: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'delivered', channel = 'C1', message_ts = '111.222', + delivered_as = 'root', delivered_at = ? WHERE id = 'ni-tb-uniq-root-1' + `, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + t.Fatalf("mark first root_sync delivered: %v", err) + } + root2 := legalRootSyncIntent("ni-tb-uniq-root-2", "sit-ni-tb-uniq", "tr-ni-tb-uniq", 1, 2) + if err := insertNotificationIntent(ctx, s, root2); err != nil { + t.Fatalf("expected a second root_sync at the same sequence to be accepted: %v", err) + } +} + +// TestSituationNotificationSchema_OnlyRootSyncMayBeSuperseded proves +// thread_append/broadcast_handoff rows can never become superseded. +func TestSituationNotificationSchema_OnlyRootSyncMayBeSuperseded(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-supersede", "group-ni-supersede", "tr-ni-supersede") + + thread := notificationIntentRow{ + id: "ni-supersede-thread", idempotencyKey: "idem-ni-supersede-thread", effectClass: "thread_append", + situationID: "sit-ni-supersede", transitionID: "tr-ni-supersede", transitionSequence: 1, + requiresRoot: 1, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, thread); err != nil { + t.Fatalf("insert thread_append: %v", err) + } + // bystander is a second, unrelated row used only as a legal (non-self) + // replacement_intent_id target, so the rejection below is attributable + // to the "only root_sync may be superseded" CHECK and not the separate + // no-self-reference CHECK. + bystander := legalRootSyncIntent("ni-supersede-bystander", "sit-ni-supersede", "tr-ni-supersede", 1, 1) + if err := insertNotificationIntent(ctx, s, bystander); err != nil { + t.Fatalf("insert bystander root_sync: %v", err) + } + // Mark it delivered so it no longer occupies the situation's + // one-pending-root_sync slot (needed below for `root`). + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'delivered', channel = 'C1', message_ts = '111.222', + delivered_as = 'root', delivered_at = ? WHERE id = ? + `, time.Now().UTC().Format(time.RFC3339Nano), bystander.id); err != nil { + t.Fatalf("mark bystander delivered: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'superseded', supersession_reason = 'x', replacement_intent_id = ? WHERE id = ? + `, bystander.id, thread.id); err == nil { + t.Fatal("expected a thread_append row to reject becoming superseded") + } + + // A root_sync that already delivered cannot retroactively become + // superseded either — only a still-pending one may. + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'superseded', supersession_reason = 'x', replacement_intent_id = ? WHERE id = ? + `, thread.id, bystander.id); err == nil { + t.Fatal("expected an already-delivered root_sync to reject becoming superseded") + } + + root := legalRootSyncIntent("ni-supersede-root", "sit-ni-supersede", "tr-ni-supersede", 1, 1) + if err := insertNotificationIntent(ctx, s, root); err != nil { + t.Fatalf("insert root_sync: %v", err) + } + // A root_sync, unlike thread_append, accepts becoming superseded. + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'superseded', supersession_reason = 'newer_root', replacement_intent_id = ? WHERE id = ? + `, bystander.id, root.id); err != nil { + t.Fatalf("expected a root_sync row to accept becoming superseded: %v", err) + } +} + +// TestSituationNotificationSchema_NoDelete proves notification_intents +// rows can never be deleted, even a terminal one. +func TestSituationNotificationSchema_NoDelete(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-delete", "group-ni-delete", "tr-ni-delete") + + row := legalRootSyncIntent("ni-delete", "sit-ni-delete", "tr-ni-delete", 1, 1) + if err := insertNotificationIntent(ctx, s, row); err != nil { + t.Fatalf("insert intent: %v", err) + } + if _, err := s.db.ExecContext(ctx, `DELETE FROM notification_intents WHERE id = ?`, row.id); err == nil { + t.Fatal("expected deleting a notification intent to be rejected") + } + + // A status mutation (the intent's ordinary lifecycle), by contrast, must + // succeed — "no DELETE" is not "no UPDATE". + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'delivered', channel = 'C1', message_ts = '111.222', + delivered_as = 'root', delivered_at = ? WHERE id = ? + `, time.Now().UTC().Format(time.RFC3339Nano), row.id); err != nil { + t.Fatalf("expected a lifecycle status update to succeed: %v", err) + } +} + +// TestSituationNotificationSchema_IdentityImmutable proves an intent's +// identity columns (effect class, subject references, poke/priority, +// deadline, client message id, created_at) can never change post-insert, +// while status/claim/retry/delivery columns can. +func TestSituationNotificationSchema_IdentityImmutable(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-immut", "group-ni-immut", "tr-ni-immut") + + row := legalRootSyncIntent("ni-immut", "sit-ni-immut", "tr-ni-immut", 1, 1) + if err := insertNotificationIntent(ctx, s, row); err != nil { + t.Fatalf("insert intent: %v", err) + } + + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET effect_class = 'thread_append' WHERE id = ?`, row.id); err == nil { + t.Fatal("expected changing effect_class to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE notification_intents SET idempotency_key = 'changed' WHERE id = ?`, row.id); err == nil { + t.Fatal("expected changing idempotency_key to be rejected") + } + + if _, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET claim_owner = 'worker-1', lease_expires_at = ?, status = 'pending' WHERE id = ? + `, time.Now().UTC().Format(time.RFC3339Nano), row.id); err != nil { + t.Fatalf("expected claiming (a lifecycle field) to succeed: %v", err) + } +} + +// TestSituationNotificationSchema_ForeignKeyShape proves a notification +// intent must reference a real Situation, Transition, and (for +// installation_gap_recovery) gap generation. +func TestSituationNotificationSchema_ForeignKeyShape(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + seedNotificationIntentFixture(ctx, t, s, "sit-ni-fk", "group-ni-fk", "tr-ni-fk") + + badSituation := legalRootSyncIntent("ni-fk-bad-situation", "does-not-exist", "tr-ni-fk", 1, 1) + if err := insertNotificationIntent(ctx, s, badSituation); err == nil { + t.Fatal("expected a nonexistent situation_id to be rejected") + } + + badTransition := legalRootSyncIntent("ni-fk-bad-transition", "sit-ni-fk", "does-not-exist", 1, 1) + if err := insertNotificationIntent(ctx, s, badTransition); err == nil { + t.Fatal("expected a nonexistent transition_id to be rejected") + } + + mismatchedSequence := legalRootSyncIntent("ni-fk-mismatch-seq", "sit-ni-fk", "tr-ni-fk", 99, 1) + if err := insertNotificationIntent(ctx, s, mismatchedSequence); err == nil { + t.Fatal("expected a transition_sequence not matching the referenced transition's actual sequence to be rejected") + } + + badGap := notificationIntentRow{ + id: "ni-fk-bad-gap", idempotencyKey: "idem-ni-fk-bad-gap", effectClass: "installation_gap_recovery", + gapGeneration: "does-not-exist", requiresRoot: 0, mainChannelPoke: 0, + } + if err := insertNotificationIntent(ctx, s, badGap); err == nil { + t.Fatal("expected a nonexistent gap_generation to be rejected") + } +} + +// ---------------------------------------------------------------------- +// Step 3: slack_delivery_state and slack_delivery_gaps. +// ---------------------------------------------------------------------- + +// insertGapGeneration seeds a minimal slack_delivery_gaps row in the given +// status and returns nothing (id is caller-supplied so FK tests can target +// it deterministically). +func insertGapGeneration(ctx context.Context, s *Store, id, status string) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + var recoveredAt, completedAt any + if status == "replaying" || status == "complete" { + recoveredAt = now + } + if status == "complete" { + completedAt = now + } + _, err := s.db.ExecContext(ctx, ` + INSERT INTO slack_delivery_gaps (id, status, opened_at, recovered_at, completed_at) + VALUES (?, ?, ?, ?, ?) + `, id, status, now, recoveredAt, completedAt) + return err +} + +// TestSlackDeliveryGapSchema_SingletonState proves slack_delivery_state +// seeds exactly one row at id=1 and rejects a second. +func TestSlackDeliveryGapSchema_SingletonState(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + + var count int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM slack_delivery_state`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("slack_delivery_state count = %d, want 1 (seeded singleton)", count) + } + + if _, err := s.db.ExecContext(ctx, ` + INSERT INTO slack_delivery_state (id, configuration_generation, updated_at) VALUES (2, 0, ?) + `, time.Now().UTC().Format(time.RFC3339Nano)); err == nil { + t.Fatal("expected a second slack_delivery_state row (id != 1) to be rejected") + } + + now := time.Now().UTC().Format(time.RFC3339Nano) + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state SET first_failure_at = ?, last_warning_at = ?, updated_at = ? WHERE id = 1 + `, now, now, now); err != nil { + t.Fatalf("expected updating the singleton to succeed: %v", err) + } + + if _, err := s.db.ExecContext(ctx, `DELETE FROM slack_delivery_state WHERE id = 1`); err == nil { + t.Fatal("expected deleting the singleton to be rejected") + } +} + +// TestSlackDeliveryGapSchema_StatusesAndFields proves the closed +// open|replaying|complete status set and the fields each carries. +func TestSlackDeliveryGapSchema_StatusesAndFields(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + + if err := insertGapGeneration(ctx, s, "gap-fields-bad", "bogus"); err == nil { + t.Fatal("expected an unknown gap status to be rejected") + } + + if err := insertGapGeneration(ctx, s, "gap-fields-open", "open"); err != nil { + t.Fatalf("expected an open gap to be accepted: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_gaps SET affected_situation_count = 3, delayed_effect_count = 5 WHERE id = 'gap-fields-open' + `); err != nil { + t.Fatalf("expected setting affected/delayed counts to succeed: %v", err) + } + + if err := insertGapGeneration(ctx, s, "gap-fields-replaying", "replaying"); err != nil { + t.Fatalf("expected a replaying gap (recovered_at set) to be accepted: %v", err) + } + + if err := insertGapGeneration(ctx, s, "gap-fields-complete", "complete"); err != nil { + t.Fatalf("expected a complete gap (recovered_at and completed_at set) to be accepted: %v", err) + } +} + +// TestSlackDeliveryGapSchema_OpenGapCannotClaimCompleteGeneration proves +// slack_delivery_state.open_gap_generation can never point at an +// already-complete generation. +func TestSlackDeliveryGapSchema_OpenGapCannotClaimCompleteGeneration(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + + if err := insertGapGeneration(ctx, s, "gap-claim-complete", "complete"); err != nil { + t.Fatalf("seed complete gap: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state SET open_gap_generation = 'gap-claim-complete', updated_at = ? WHERE id = 1 + `, time.Now().UTC().Format(time.RFC3339Nano)); err == nil { + t.Fatal("expected claiming a complete generation as open to be rejected") + } + + if err := insertGapGeneration(ctx, s, "gap-claim-open", "open"); err != nil { + t.Fatalf("seed open gap: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state SET open_gap_generation = 'gap-claim-open', updated_at = ? WHERE id = 1 + `, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + t.Fatalf("expected claiming an open generation to succeed: %v", err) + } + + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state SET open_gap_generation = ?, updated_at = ? WHERE id = 1 + `, nil, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + t.Fatalf("expected clearing open_gap_generation to succeed: %v", err) + } + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state SET open_gap_generation = ?, updated_at = ? WHERE id = 1 + `, "does-not-exist", time.Now().UTC().Format(time.RFC3339Nano)); err == nil { + t.Fatal("expected claiming a nonexistent generation to be rejected") + } +} + +// TestSlackDeliveryGapSchema_NoDelete proves slack_delivery_gaps rows are +// never deleted, and identity (id, opened_at) never changes post-insert. +func TestSlackDeliveryGapSchema_NoDelete(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + + if err := insertGapGeneration(ctx, s, "gap-nodelete", "open"); err != nil { + t.Fatalf("seed gap: %v", err) + } + if _, err := s.db.ExecContext(ctx, `DELETE FROM slack_delivery_gaps WHERE id = 'gap-nodelete'`); err == nil { + t.Fatal("expected deleting a gap generation to be rejected") + } + if _, err := s.db.ExecContext(ctx, `UPDATE slack_delivery_gaps SET id = 'renamed' WHERE id = 'gap-nodelete'`); err == nil { + t.Fatal("expected changing a gap generation's id to be rejected") + } + + now := time.Now().UTC().Format(time.RFC3339Nano) + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_gaps SET status = 'replaying', recovered_at = ? WHERE id = 'gap-nodelete' + `, now); err != nil { + t.Fatalf("expected the ordinary open->replaying lifecycle update to succeed: %v", err) + } +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 51555ed..331a134 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -58,6 +58,9 @@ func TestOpen_AppliesEmbeddedMigrations(t *testing.T) { "situation_transitions": false, "situation_episode_summaries": false, "situation_transition_stream": false, + "notification_intents": false, + "slack_delivery_state": false, + "slack_delivery_gaps": false, } for rows.Next() { var name string @@ -443,12 +446,12 @@ func TestMaxSchemaVersion(t *testing.T) { if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - // 0017_situation_history.sql is the newest migration today. Plan 2 owns - // 0015/0016; Plan 3 owns exactly 0017/0018 (spec.md "Persistence and - // migration ownership") and this task lands 0017, so the number moves - // from 16 to 17 — 0018 (notification_intents et al.) is a later task. - if got != 17 { - t.Errorf("MaxSchemaVersion = %d, want 17", got) + // 0018_situation_notifications.sql is the newest migration today. Plan 2 + // owns 0015/0016; Plan 3 owns exactly 0017/0018 (spec.md "Persistence + // and migration ownership") and both now land, so the number moves from + // 17 to 18 — this is Plan 3's final schema migration. + if got != 18 { + t.Errorf("MaxSchemaVersion = %d, want 18", got) } } From cf92b40629bed294e863678341753057be500643 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 00:54:21 +0300 Subject: [PATCH 04/31] fix(store): update MaxSchemaVersion assertion for migration 0018 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 2's history-upgrade test hardcoded MaxSchemaVersion() == 17, which was correct only while 0017 was the newest migration. Task 3 legitimately bumps the global max to 18 (owned by store_test.go's TestMaxSchemaVersion), so drop the redundant exact-value assertion here — migration 17 landing is already proven by the schema_migrations row-count check two lines above. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- .../store/situation_history_upgrade_test.go | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/store/situation_history_upgrade_test.go b/internal/store/situation_history_upgrade_test.go index 5d5b77b..b935deb 100644 --- a/internal/store/situation_history_upgrade_test.go +++ b/internal/store/situation_history_upgrade_test.go @@ -15,9 +15,10 @@ import ( // ---------------------------------------------------------------------- // Step 1: migration 0017 upgrade tests — a populated Plan 2 (migration 16) -// fixture must gain the new STRICT tables and MaxSchemaVersion 17, pass -// PRAGMA foreign_key_check, and acquire zero fabricated Transition history -// for its pre-existing nonterminal/terminal Situations. +// fixture must gain the new STRICT tables, land migration 17 in +// schema_migrations, pass PRAGMA foreign_key_check, and acquire zero +// fabricated Transition history for its pre-existing nonterminal/terminal +// Situations. // ---------------------------------------------------------------------- // seedMigration16HistoryFixture builds a database file shaped like the @@ -112,10 +113,17 @@ func seedMigration16HistoryFixture(t *testing.T, path string) (nonterminalID, te // TestSituationHistoryUpgrade_CreatesStrictTablesAndBumpsSchemaVersion is // the brief's literal Step 1 test: opening a migration-16 database with the -// current Open must apply 0017, create its three new STRICT tables, bump -// MaxSchemaVersion to 17, pass PRAGMA foreign_key_check, and leave the -// fixture's pre-existing nonterminal/terminal Situations with zero +// current Open must apply 0017, create its three new STRICT tables, land +// migration 17 in schema_migrations, pass PRAGMA foreign_key_check, and +// leave the fixture's pre-existing nonterminal/terminal Situations with zero // Transitions. +// +// This intentionally does not also assert MaxSchemaVersion()'s exact value: +// that is a global fact about every embedded migration, owned by +// store_test.go's dedicated TestMaxSchemaVersion, not by any one migration's +// own upgrade test — asserting an exact global max here would go stale the +// moment a later task (0018 onward) adds another migration, exactly as +// happened once Task 3 landed 0018. func TestSituationHistoryUpgrade_CreatesStrictTablesAndBumpsSchemaVersion(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "migration16-history.db") @@ -135,14 +143,6 @@ func TestSituationHistoryUpgrade_CreatesStrictTablesAndBumpsSchemaVersion(t *tes t.Fatalf("migration 17 applied count = %d, want 1", applied) } - got, err := MaxSchemaVersion() - if err != nil { - t.Fatalf("MaxSchemaVersion: %v", err) - } - if got != 17 { - t.Fatalf("MaxSchemaVersion = %d, want 17", got) - } - for _, table := range []string{"situation_transitions", "situation_episode_summaries", "situation_transition_stream"} { var name string if err := st.DB().QueryRowContext(ctx, `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&name); err != nil { From 86b8ea4a7a7becfb8e6957f6b2cf3ec351d892a4 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 01:39:55 +0300 Subject: [PATCH 05/31] feat(situation): derive durable operator history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BuildTransitions, ProjectEpisode, PlanNotificationIntents, and the BuildHistoryCommit composition Task 5 commits atomically. Two fixes in Task 1's model/history.go were required to land this: - ProjectionFacts.Validate rejected terminal_at without terminal_reason, which made every `recovered` Transition unrepresentable — migration 0014's own lifecycle CHECK records recovered as terminal_at NOT NULL with terminal_reason NULL, and TerminalReason's closed vocabulary only describes a closed_unknown closure. - NotificationIntent.Validate exceeded the repo's gocyclo limit (lint was already red on this branch); its bounded-reference and effect-class reference checks are now two helpers. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/situation/history.go | 1103 ++++++++++++++ internal/situation/history_test.go | 1426 ++++++++++++++++++ internal/situation/model/history.go | 117 +- internal/situation/model/history_test.go | 11 +- internal/situation/notification_plan.go | 289 ++++ internal/situation/notification_plan_test.go | 655 ++++++++ internal/situation/priority.go | 136 ++ internal/situation/priority_test.go | 266 ++++ 8 files changed, 3955 insertions(+), 48 deletions(-) create mode 100644 internal/situation/history.go create mode 100644 internal/situation/history_test.go create mode 100644 internal/situation/notification_plan.go create mode 100644 internal/situation/notification_plan_test.go create mode 100644 internal/situation/priority.go create mode 100644 internal/situation/priority_test.go diff --git a/internal/situation/history.go b/internal/situation/history.go new file mode 100644 index 0000000..cd43f99 --- /dev/null +++ b/internal/situation/history.go @@ -0,0 +1,1103 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 4: the pure derivation of one controller reconciliation's +// durable operator history — the immutable Transitions it creates +// (BuildTransitions), the Episode-summary projection folded across them +// (ProjectEpisode), and the composition Task 5 commits atomically +// (BuildHistoryCommit). No I/O, no clock read beyond the explicit Now, no +// Slack rendering: this package decides what became true, never how it +// looks. +// ---------------------------------------------------------------------- + +// Bounded lengths mirroring internal/situation/model's own (unexported) +// bounds for the same fields, so a value derived here can never be +// rejected by model.Transition.Validate for length alone. +const ( + maxHistoryHeadline = 200 + maxHistoryDetail = 2000 + maxHistoryIdentifier = 200 + maxHistoryEvidenceRefs = 50 + // maxHistoryListEntries bounds the two accumulating Episode-summary + // lists (investigation work, recorded operator context). The immutable + // Transition ledger remains the complete record; the summary is a + // bounded current projection. + maxHistoryListEntries = 50 +) + +// The two durable operator-artifact input kinds (R5). They mirror +// internal/store's isOperatorArtifactKind exactly. +const ( + artifactKindAnnotation = "operator_annotation_recorded" + artifactKindVerdict = "captured_verdict_recorded" +) + +// Acute Triage schedule phases this package reads (migration 0016). Only +// the terminal skip phase needs naming here: it is the pre-claim clean skip +// (TriageStore.CleanSkipIncidentTriageBelowMinimumMembers) that consumes no +// attempt and must never read as investigation having run. +const triagePhaseSkipped = "skipped" + +// historyNamespace is the fixed AlertINT UUID namespace every deterministic +// Plan 3 identity (Transition ID, notification intent ID, Slack client +// message ID) is derived under, so the same inputs always produce the same +// identity across retries, restarts, and processes. +var historyNamespace = uuid.NewSHA1(uuid.NameSpaceURL, []byte("https://alertint.com/ns/situation-history/v1")) + +// AuthoritativeChange is one fenced controller reconciliation's +// authoritative result, reduced to exactly what deriving durable history +// needs. Situation carries the COMMITTED projection (lifecycle, Attention, +// recovery/terminal instants) — never the pre-commit read. +type AuthoritativeChange struct { + Situation model.Situation + AssessmentID *string + Assessment model.Assessment + // Derivation is how the authoritative Assessment came to be. It decides + // the Transition actor for the reasons whose basis is model-authored + // content: only a model_validated Assessment may be recorded as `llm`. + Derivation model.AssessmentDerivation + // Projection is captured from the coherent claim and the commit's + // lifecycle fields (R3) — the only thing ProjectEpisode may read. + Projection model.ProjectionFacts + PriorTransition *model.Transition + PriorSummary *model.EpisodeSummary + MaterialFactHash string + EvidenceRefs []string + // Incidents is Plan 2's per-Incident state including IncidentState.Triage + // (R7); TriageDecisions is this cycle's Plan 2 request/skip decisions. + Incidents []IncidentState + TriageDecisions []TriageDecision + RecurrenceCount int + OperatorArtifacts []OperatorArtifactInput // R1: every applied-and-unjournaled artifact, in order + Drill bool + Now time.Time +} + +// OperatorArtifactInput is one applied-and-unjournaled durable operator +// artifact input (R1). The caller orders these by +// (applied_input_version, occurred_at, id); BuildTransitions preserves that +// order exactly. +type OperatorArtifactInput struct { + InputID string // situation_input_outbox.id — becomes Transition.OperatorArtifactInputID + Kind string // operator_annotation_recorded | captured_verdict_recorded + AnnotationID *string + VerdictID *string + AppliedInputVersion int + OccurredAt time.Time + AttributedActor string + Headline string // bounded, from the durable artifact + Detail string // bounded +} + +// HistoryCommit is everything one fenced controller transaction commits on +// top of Plan 2's authoritative result. +type HistoryCommit struct { + // Transitions is this reconciliation's immutable history in sequence + // order: consumed operator artifacts first, the controller-state + // Transition last (R1). Empty when nothing material happened. + Transitions []model.Transition + // Summary is the Episode projection after folding every Transition in + // order; nil when Transitions is empty. + Summary *model.EpisodeSummary + Intents []model.NotificationIntent +} + +// ---------------------------------------------------------------------- +// BuildTransitions +// ---------------------------------------------------------------------- + +// BuildTransitions derives one reconciliation's immutable Transitions: one +// `operator_artifact_recorded` Transition per pending artifact in the given +// order, then at most one controller-state Transition when the semantic +// tuple actually changed (R1, R4). +// +// Materiality is semantic. The compared tuple is lifecycle, Attention, the +// Operator contract WITHOUT its next_update_at, the accepted Sufficient +// reason's code, the Assessment's closed conclusion codes, per-Incident +// Triage phase/decision/result, and the recurrence milestone. Plan 2's +// per-input-version identities — Assessment IDs, reason-candidate IDs, +// fact/evidence IDs, the material fact hash — and the refreshed +// next_update_at are deliberately excluded, so a `revalidated_reuse` cycle +// adds no history. +// +// Reason precedence for the single controller-state Transition, most +// operator-consequential first: +// +// first_authoritative_state > recovered > closed_unknown > +// recovery_failed > recovery_observed > investigation_concluded > +// investigation_started > attention_changed > operator_contract_changed > +// recurrence_milestone > triage_state_changed > material_assessment_changed +// +// The selected reason names the change; the Transition still records the +// complete state and every supporting evidence reference. +func BuildTransitions(change AuthoritativeChange) ([]model.Transition, error) { + if err := validateChange(change); err != nil { + return nil, err + } + + out := make([]model.Transition, 0, len(change.OperatorArtifacts)+1) + base := 0 + if change.PriorTransition != nil { + base = change.PriorTransition.Sequence + } + + for i, artifact := range change.OperatorArtifacts { + tr, err := artifactTransition(change, artifact, base+1+i) + if err != nil { + return nil, fmt.Errorf("situation: operator artifact %d: %w", i, err) + } + out = append(out, tr) + } + + if reason, material := selectControllerReason(change); material { + out = append(out, controllerTransition(change, reason, base+1+len(out))) + } + + for i := range out { + if err := out[i].Validate(); err != nil { + return nil, fmt.Errorf("situation: derived transition %d: %w", i, err) + } + } + return out, nil +} + +func validateChange(change AuthoritativeChange) error { + if strings.TrimSpace(change.Situation.ID) == "" { + return errors.New("situation: authoritative change: situation id is required") + } + if change.Situation.InputVersion < 1 { + return fmt.Errorf("situation: authoritative change: input version must be >= 1, got %d", change.Situation.InputVersion) + } + if strings.TrimSpace(change.MaterialFactHash) == "" { + return errors.New("situation: authoritative change: material fact hash is required") + } + if change.Now.IsZero() || change.Now.Location() != time.UTC { + return fmt.Errorf("situation: authoritative change: now must be a non-zero UTC instant, got %s", change.Now) + } + if change.RecurrenceCount < 0 { + return fmt.Errorf("situation: authoritative change: recurrence count must be >= 0, got %d", change.RecurrenceCount) + } + if err := change.Situation.Lifecycle.Validate(); err != nil { + return fmt.Errorf("situation: authoritative change: %w", err) + } + if err := change.Situation.Attention.Validate(); err != nil { + return fmt.Errorf("situation: authoritative change: %w", err) + } + if change.Derivation != "" { + if err := change.Derivation.Validate(); err != nil { + return fmt.Errorf("situation: authoritative change: %w", err) + } + } + // The Transition records the committed lifecycle and the projection + // captured for the same commit; they must agree, or the durable record + // would contradict itself. + terminal := change.Situation.Lifecycle.Terminal() + if terminal != (change.Projection.TerminalAt != nil) { + return fmt.Errorf("situation: authoritative change: lifecycle %q and projection terminal_at %v disagree", + change.Situation.Lifecycle, change.Projection.TerminalAt) + } + if err := change.Projection.Validate(); err != nil { + return fmt.Errorf("situation: authoritative change: %w", err) + } + if change.PriorTransition != nil { + if change.PriorTransition.SituationID != change.Situation.ID { + return fmt.Errorf("situation: authoritative change: prior transition belongs to situation %q, not %q", + change.PriorTransition.SituationID, change.Situation.ID) + } + if change.PriorSummary == nil { + return errors.New("situation: authoritative change: a prior transition requires the prior episode summary") + } + if change.PriorSummary.SituationID != change.Situation.ID { + return fmt.Errorf("situation: authoritative change: prior summary belongs to situation %q, not %q", + change.PriorSummary.SituationID, change.Situation.ID) + } + } + return nil +} + +// artifactTransition records one durable operator artifact. It carries the +// commit's current controller state verbatim: an annotation or Captured +// verdict adds journal context and never alters the Assessment, Attention, +// or Operator contract (spec.md "Situation journal entry"). +func artifactTransition(change AuthoritativeChange, artifact OperatorArtifactInput, sequence int) (model.Transition, error) { + if strings.TrimSpace(artifact.InputID) == "" { + return model.Transition{}, errors.New("input id is required") + } + if artifact.OccurredAt.IsZero() || artifact.OccurredAt.Location() != time.UTC { + return model.Transition{}, fmt.Errorf("occurred_at must be a non-zero UTC instant, got %s", artifact.OccurredAt) + } + var journalKind model.JournalKind + var evidence []string + switch artifact.Kind { + case artifactKindAnnotation: + if artifact.AnnotationID == nil || strings.TrimSpace(*artifact.AnnotationID) == "" { + return model.Transition{}, errors.New("operator_annotation_recorded requires an annotation id") + } + if artifact.VerdictID != nil { + return model.Transition{}, errors.New("operator_annotation_recorded must not carry a verdict id") + } + journalKind = model.JournalOperatorNote + evidence = []string{"annotation:" + *artifact.AnnotationID} + case artifactKindVerdict: + if artifact.VerdictID == nil || strings.TrimSpace(*artifact.VerdictID) == "" { + return model.Transition{}, errors.New("captured_verdict_recorded requires a verdict id") + } + if artifact.AnnotationID != nil { + return model.Transition{}, errors.New("captured_verdict_recorded must not carry an annotation id") + } + journalKind = model.JournalCapturedVerdict + evidence = []string{"verdict:" + *artifact.VerdictID} + default: + return model.Transition{}, fmt.Errorf("unsupported operator artifact kind %q", artifact.Kind) + } + + tr := newTransition(change, sequence, model.ReasonOperatorArtifactRecorded, artifact.InputID, journalKind, model.JournalData{ + Headline: boundedText(artifact.Headline, maxHistoryHeadline), + Detail: boundedText(artifact.Detail, maxHistoryDetail), + AttributedActor: boundedText(artifact.AttributedActor, maxHistoryIdentifier), + ActionStatus: contractActionStatus(change.Assessment.ActionContract), + RecurrenceCount: change.RecurrenceCount, + OccurredAt: artifact.OccurredAt, + }, evidence) + tr.Actor = model.ActorAttributedOperator + tr.OperatorArtifactInputID = stringPtrOf(artifact.InputID) + return tr, nil +} + +func controllerTransition(change AuthoritativeChange, reason model.TransitionReason, sequence int) model.Transition { + journalKind := journalKindFor(reason) + headline, detail := journalTextFor(change, reason) + tr := newTransition(change, sequence, reason, "", journalKind, model.JournalData{ + Headline: boundedText(headline, maxHistoryHeadline), + Detail: boundedText(detail, maxHistoryDetail), + ActionStatus: contractActionStatus(change.Assessment.ActionContract), + RecurrenceCount: change.RecurrenceCount, + OccurredAt: change.Now, + }, controllerEvidenceRefs(change)) + tr.Actor = controllerActor(change, reason) + if class := ClassifyPoke(change.PriorTransition, tr); class != PokeNone { + priority := DeriveInterruptionPriority(tr) + tr.InterruptionPriority = &priority + } + return tr +} + +// newTransition fills every field a Transition shares regardless of reason: +// deterministic identity, the committed controller state, this commit's +// captured projection facts (R3), and the Drill marker. +func newTransition(change AuthoritativeChange, sequence int, reason model.TransitionReason, + artifactInputID string, journalKind model.JournalKind, journal model.JournalData, evidence []string) model.Transition { + tr := model.Transition{ + SituationID: change.Situation.ID, + Sequence: sequence, + InputVersion: change.Situation.InputVersion, + MaterialFactHash: change.MaterialFactHash, + AssessmentID: change.AssessmentID, + Lifecycle: change.Situation.Lifecycle, + Attention: change.Situation.Attention, + ActionContract: change.Assessment.ActionContract, + Reason: reason, + JournalKind: journalKind, + Journal: journal, + Projection: change.Projection, + EvidenceRefs: evidence, + Actor: model.ActorDeterministicController, + Drill: change.Drill, + CreatedAt: change.Now, + } + if change.Assessment.SufficientReason != nil && change.Assessment.SufficientReason.CandidateID != "" { + tr.SufficientReasonID = stringPtrOf(change.Assessment.SufficientReason.CandidateID) + } + tr.ID = transitionIdentity(change.Situation.ID, change.Situation.InputVersion, sequence, reason, artifactInputID) + return tr +} + +// transitionIdentitySchemaVersion versions the canonical hash input below; +// bump it only when the identity fields themselves change meaning. +const transitionIdentitySchemaVersion = 1 + +type transitionIdentityDTO struct { + SchemaVersion int `json:"schema_version"` + SituationID string `json:"situation_id"` + InputVersion int `json:"input_version"` + Sequence int `json:"sequence"` + Reason string `json:"reason"` + Artifact string `json:"artifact,omitempty"` +} + +// transitionIdentity derives a Transition's stable UUIDv5 identity from the +// Situation, the authoritative input version, the Transition's own sequence +// and reason, and (for an artifact Transition) the artifact it journals. +// Two runs of the same fenced commit therefore produce byte-identical +// identities, and a later input version never collides with an earlier one. +func transitionIdentity(situationID string, inputVersion, sequence int, reason model.TransitionReason, artifact string) string { + return uuid.NewSHA1(historyNamespace, mustMarshal(transitionIdentityDTO{ + SchemaVersion: transitionIdentitySchemaVersion, + SituationID: situationID, + InputVersion: inputVersion, + Sequence: sequence, + Reason: string(reason), + Artifact: artifact, + })).String() +} + +// controllerActor records `llm` only for the reasons whose basis is +// model-authored content (the conclusion codes, Attention, and the selected +// Sufficient reason), and only when the authoritative Assessment was +// actually model-validated. Lifecycle, the Operator contract, Triage state, +// and recurrence are controller-derived, so they are always +// deterministic_controller. Plan 3 never produces `operator_policy`. +func controllerActor(change AuthoritativeChange, reason model.TransitionReason) model.TransitionActor { + if change.Derivation != model.DerivationModelValidated { + return model.ActorDeterministicController + } + switch reason { //nolint:exhaustive // the default is the point: every other reason records controller-derived lifecycle, contract, Triage, or recurrence state, which is never the model's authorship. + case model.ReasonFirstAuthoritativeState, model.ReasonMaterialAssessmentChanged, + model.ReasonAttentionChanged, model.ReasonInvestigationConcluded: + return model.ActorLLM + default: + return model.ActorDeterministicController + } +} + +// controllerEvidenceRefs retains every supporting evidence reference for +// this commit: the caller's collected references plus the accepted +// Sufficient reason's own, deduplicated and ordered so the durable record +// is canonical. +func controllerEvidenceRefs(change AuthoritativeChange) []string { + refs := append([]string{}, change.EvidenceRefs...) + if change.Assessment.SufficientReason != nil { + refs = append(refs, change.Assessment.SufficientReason.EvidenceRefs...) + } + return canonicalRefs(refs) +} + +func canonicalRefs(refs []string) []string { + seen := make(map[string]bool, len(refs)) + out := make([]string, 0, len(refs)) + for _, r := range refs { + r = boundedText(strings.TrimSpace(r), maxHistoryIdentifier) + if r == "" || seen[r] { + continue + } + seen[r] = true + out = append(out, r) + } + sort.Strings(out) + if len(out) > maxHistoryEvidenceRefs { + out = out[:maxHistoryEvidenceRefs] + } + return out +} + +// ---------------------------------------------------------------------- +// Materiality (R4) +// ---------------------------------------------------------------------- + +// materialTuple is the canonical semantic identity of one committed +// controller state. Everything in it is a closed code or a bounded closed +// list; no ID, hash, prose, or instant appears. +type materialTuple struct { + Lifecycle string `json:"lifecycle"` + Attention string `json:"attention"` + Contract string `json:"contract"` + HasAssessment bool `json:"has_assessment"` + Persistence string `json:"persistence"` + Impact string `json:"impact"` + Novelty string `json:"novelty"` + Causality string `json:"causality"` + EvidenceQuality string `json:"evidence_quality"` + LimitationCodes []string `json:"limitation_codes"` + SufficientReason string `json:"sufficient_reason_code"` + RecurrenceMilestone int `json:"recurrence_milestone"` +} + +// operatorContractTuple canonicalizes the Operator contract WITHOUT its +// next_update_at: the deadline instant reaches the root through the +// intent's ContractDeadlineAt (R4), never by making every cadence tick +// material. +func operatorContractTuple(c model.ActionContract) string { + on := make([]string, 0, len(c.NextUpdateOn)) + for _, o := range c.NextUpdateOn { + on = append(on, string(o)) + } + sort.Strings(on) + return canonicalDigest(struct { + NextActor string `json:"next_actor"` + AlertINTAction string `json:"alertint_action"` + AlertINTStatus string `json:"alertint_status"` + OperatorAction string `json:"operator_action_required"` + NextUpdateOn []string `json:"next_update_on"` + WaitReason string `json:"wait_reason"` + }{ + NextActor: string(c.NextActor), + AlertINTAction: derefAlertINTAction(c.AlertINTAction), + AlertINTStatus: derefAlertINTStatus(c.AlertINTStatus), + OperatorAction: derefOperatorAction(c.OperatorActionRequired), + NextUpdateOn: on, + WaitReason: derefWaitReason(c.WaitReason), + }) +} + +func tupleOf(lifecycle model.Lifecycle, attention model.Attention, contract model.ActionContract, + concl *model.AssessmentConclusion, recurrence int) materialTuple { + t := materialTuple{ + Lifecycle: string(lifecycle), + Attention: string(attention), + Contract: operatorContractTuple(contract), + LimitationCodes: []string{}, + RecurrenceMilestone: recurrenceMilestone(recurrence), + } + if concl != nil { + codes := append([]string{}, concl.LimitationCodes...) + sort.Strings(codes) + t.HasAssessment = true + t.Persistence = string(concl.Persistence) + t.Impact = string(concl.Impact) + t.Novelty = string(concl.Novelty) + t.Causality = string(concl.Causality) + t.EvidenceQuality = string(concl.EvidenceQuality) + t.LimitationCodes = codes + t.SufficientReason = concl.SufficientReasonCode + } + return t +} + +func currentTuple(change AuthoritativeChange) materialTuple { + return tupleOf(change.Situation.Lifecycle, change.Situation.Attention, + change.Assessment.ActionContract, change.Projection.Assessment, change.RecurrenceCount) +} + +func priorTuple(change AuthoritativeChange) materialTuple { + prior := change.PriorTransition + recurrence := 0 + if change.PriorSummary != nil { + recurrence = change.PriorSummary.RecurrenceCount + } + return tupleOf(prior.Lifecycle, prior.Attention, prior.ActionContract, prior.Projection.Assessment, recurrence) +} + +// recurrenceMilestone reduces a raw recurrence count to the sparse, +// roughly logarithmic milestone rung the legacy presentation path already +// used (ADR-0020: x5, x10, x25, x50, then every x100), and 0 below the +// first rung. The tuple carries the RUNG, not the raw count, so a flapper +// creates a bounded handful of Transitions instead of one per re-fire — +// spec.md: "A silent Situation has no Slack recurrence trace." The raw +// count still reaches the Episode summary through the journal. +func recurrenceMilestone(count int) int { + switch { + case count >= 100: + return count / 100 * 100 + case count >= 50: + return 50 + case count >= 25: + return 25 + case count >= 10: + return 10 + case count >= 5: + return 5 + default: + return 0 + } +} + +// selectControllerReason applies the documented precedence over the +// semantic tuple, returning false when nothing material changed. +func selectControllerReason(change AuthoritativeChange) (model.TransitionReason, bool) { + prior := change.PriorTransition + if prior == nil { + return model.ReasonFirstAuthoritativeState, true + } + + current := currentTuple(change) + previous := priorTuple(change) + triage := triageStateChanged(change) + if canonicalDigest(current) == canonicalDigest(previous) && !triage { + return "", false + } + + priorInvestigating := investigationCurrent(prior.ActionContract) + nowInvestigating := investigationCurrent(change.Assessment.ActionContract) + investigationStarted := nowInvestigating && !priorInvestigating && + (change.PriorSummary == nil || !change.PriorSummary.InvestigationStarted) + + switch { + case change.Situation.Lifecycle == model.LifecycleRecovered: + return model.ReasonRecovered, true + case change.Situation.Lifecycle == model.LifecycleClosedUnknown: + return model.ReasonClosedUnknown, true + case prior.Lifecycle == model.LifecycleRecoveryPending && change.Situation.Lifecycle == model.LifecycleActive: + return model.ReasonRecoveryFailed, true + case change.Situation.Lifecycle == model.LifecycleRecoveryPending && prior.Lifecycle != model.LifecycleRecoveryPending: + return model.ReasonRecoveryObserved, true + case priorInvestigating && !nowInvestigating && change.Assessment.SufficientReason != nil: + return model.ReasonInvestigationConcluded, true + case investigationStarted: + return model.ReasonInvestigationStarted, true + case current.Attention != previous.Attention: + return model.ReasonAttentionChanged, true + case current.Contract != previous.Contract: + return model.ReasonOperatorContractChanged, true + case current.RecurrenceMilestone != previous.RecurrenceMilestone: + return model.ReasonRecurrenceMilestone, true + case triage: + return model.ReasonTriageStateChanged, true + default: + // Whatever is left in the tuple is the Assessment's own conclusion. + return model.ReasonMaterialAssessmentChanged, true + } +} + +// investigationCurrent reports whether the Operator contract currently +// names AlertINT investigation work — the durable basis for the Episode +// summary's InvestigationStarted flag and the root's Investigating phase. +// Monitoring and recovery verification are not investigation. +func investigationCurrent(c model.ActionContract) bool { + if c.AlertINTAction == nil { + return false + } + switch *c.AlertINTAction { //nolint:exhaustive // monitor_situation and verify_recovery are deliberately NOT investigation; they fall to the default. + case model.AlertINTActionRunAcuteTriage, model.AlertINTActionRetrySituationAssessment: + return true + default: + return false + } +} + +// triageStateChanged reports whether per-Incident Acute Triage state moved +// since the last recorded Transition. Neither the Transition ledger nor the +// Episode summary carries a prior per-Incident Triage tuple, so the change +// is established from three durable signals the reconciliation already +// carries: a decision made in this cycle, a Triage decision or completed +// attempt newer than the last recorded Transition, and the consumed +// `triage_changed` due reason (which is exactly what Plan 2's +// triage_skipped/triage_retry_changed/triage_exhausted inputs raise — +// including the pre-claim minimum-member clean skip that consumes no +// attempt). +func triageStateChanged(change AuthoritativeChange) bool { + if len(change.TriageDecisions) > 0 { + return true + } + for _, r := range change.Situation.DueReasons { + if r == model.DueTriageChanged { + return true + } + } + if change.PriorTransition == nil { + return false + } + watermark := change.PriorTransition.CreatedAt + for _, inc := range change.Incidents { + if inc.Triage.DecidedAt != nil && inc.Triage.DecidedAt.After(watermark) { + return true + } + if inc.Triage.LatestAttempt != nil && inc.Triage.LatestAttempt.CompletedAt.After(watermark) { + return true + } + } + return false +} + +// ---------------------------------------------------------------------- +// Journal render data +// ---------------------------------------------------------------------- + +func journalKindFor(reason model.TransitionReason) model.JournalKind { + switch reason { //nolint:exhaustive // the default deliberately maps both remaining controller-state reasons to investigation_changed; operator_artifact_recorded never reaches here. + case model.ReasonFirstAuthoritativeState: + return model.JournalPublication + case model.ReasonRecovered: + return model.JournalRecovered + case model.ReasonClosedUnknown: + return model.JournalClosedUnknown + case model.ReasonRecoveryFailed: + return model.JournalRecoveryRefired + case model.ReasonRecoveryObserved: + return model.JournalRecoveryPending + case model.ReasonInvestigationConcluded: + return model.JournalEvidenceConclusion + case model.ReasonInvestigationStarted: + return model.JournalInvestigationStarted + case model.ReasonAttentionChanged, model.ReasonOperatorContractChanged: + return model.JournalOperatorContractChanged + case model.ReasonRecurrenceMilestone: + return model.JournalRecurrenceMilestone + default: + // triage_state_changed and material_assessment_changed both report + // how the investigation itself moved. + return model.JournalInvestigationChanged + } +} + +func journalTextFor(change AuthoritativeChange, reason model.TransitionReason) (headline, detail string) { + summary := "" + if change.Projection.Assessment != nil { + summary = change.Projection.Assessment.SufficientReasonSummary + } + switch reason { //nolint:exhaustive // operator_artifact_recorded renders from the durable artifact itself (artifactTransition), never from here. + case model.ReasonFirstAuthoritativeState: + return "Situation published", summary + case model.ReasonMaterialAssessmentChanged: + return "Assessment conclusion changed", summary + case model.ReasonAttentionChanged: + return "Attention changed to " + string(change.Situation.Attention), summary + case model.ReasonOperatorContractChanged: + if change.Assessment.ActionContract.OperatorActionRequired != nil { + return "Operator action required: " + string(*change.Assessment.ActionContract.OperatorActionRequired), summary + } + return "Operator contract changed", summary + case model.ReasonInvestigationStarted: + return "AlertINT investigation started", contractActionDetail(change.Assessment.ActionContract) + case model.ReasonInvestigationConcluded: + return "AlertINT investigation concluded", summary + case model.ReasonRecoveryObserved: + return "Recovery observed; watching for sustained recovery", contractActionDetail(change.Assessment.ActionContract) + case model.ReasonRecoveryFailed: + return "Recovery did not hold; the condition refired", contractActionDetail(change.Assessment.ActionContract) + case model.ReasonRecovered: + return "Recovered", summary + case model.ReasonClosedUnknown: + d := summary + if change.Projection.TerminalReason != nil { + d = "Terminal reason: " + string(*change.Projection.TerminalReason) + } + return "Closed with uncertainty", d + case model.ReasonRecurrenceMilestone: + return fmt.Sprintf("Recurrence milestone: %d occurrences", change.RecurrenceCount), summary + case model.ReasonTriageStateChanged: + return triageJournalText(change) + default: + return "Situation state changed", summary + } +} + +// triageJournalText reports what happened to Acute Triage and why, in +// closed codes only. A clean skip — Plan 2's B+ decision skip, or the +// pre-claim minimum-member skip that consumes no attempt — says so +// explicitly and never renders as investigation having run. +func triageJournalText(change AuthoritativeChange) (string, string) { + for _, d := range change.TriageDecisions { + if d.Decision == TriageDecisionSkip { + return "Acute Triage skipped", "Decision: " + d.DecisionReason + } + } + for _, inc := range change.Incidents { + if inc.Triage.Phase != triagePhaseSkipped { + continue + } + if inc.Triage.LatestAttempt != nil { + return "Acute Triage skipped", "Attempt result: " + inc.Triage.LatestAttempt.ResultCode + } + return "Acute Triage skipped", "Closed as a clean skip before any attempt was claimed; no attempt was consumed." + } + for _, d := range change.TriageDecisions { + if d.Decision == TriageDecisionRequest { + return "Acute Triage requested", "Decision: " + d.DecisionReason + } + } + for _, inc := range change.Incidents { + if inc.Triage.LatestAttempt != nil { + return "Acute Triage state changed", "Attempt result: " + inc.Triage.LatestAttempt.ResultCode + } + } + return "Acute Triage state changed", "" +} + +func contractActionStatus(c model.ActionContract) string { + if c.AlertINTStatus == nil { + return "" + } + return string(*c.AlertINTStatus) +} + +func contractActionDetail(c model.ActionContract) string { + if c.AlertINTAction == nil { + return "Next actor: " + string(c.NextActor) + } + return "AlertINT action: " + string(*c.AlertINTAction) + " (" + contractActionStatus(c) + ")" +} + +// ---------------------------------------------------------------------- +// ProjectEpisode +// ---------------------------------------------------------------------- + +// ProjectEpisode folds one contiguous Transition into the durable Episode +// summary. It reads nothing but its two arguments (R3): every field comes +// from the Transition's captured projection facts and bounded journal data, +// never from mutable Incident or Situation state. Every fold advances the +// version by exactly one. +// +// It rejects a skipped or duplicated sequence, a Situation mismatch, a time +// reversal, and every mutation after a terminal Transition — a later firing +// creates a separately linked Situation through Plan 1/2 recurrence +// ownership and never reopens a terminal Episode. +func ProjectEpisode(prior *model.EpisodeSummary, t model.Transition) (model.EpisodeSummary, error) { + if err := t.Validate(); err != nil { + return model.EpisodeSummary{}, fmt.Errorf("situation: project episode: %w", err) + } + + out := model.EpisodeSummary{ + SituationID: t.SituationID, + Version: 1, + InvestigationWork: []string{}, + RecordedOperatorContext: []string{}, + PeakAttention: t.Attention, + } + if prior != nil { + if err := validateFold(*prior, t); err != nil { + return model.EpisodeSummary{}, err + } + out = *prior + out.Version = prior.Version + 1 + out.InvestigationWork = append([]string{}, prior.InvestigationWork...) + out.RecordedOperatorContext = append([]string{}, prior.RecordedOperatorContext...) + if attentionRank(t.Attention) > attentionRank(prior.PeakAttention) { + out.PeakAttention = t.Attention + } + } + + out.SourceTransitionSequence = t.Sequence + out.CurrentAttention = t.Attention + out.ActionContract = t.ActionContract + out.UpdatedAt = t.CreatedAt + out.EffectiveStartedAt = t.Projection.EffectiveStartedAt + out.RecoveryObservedAt = t.Projection.RecoveryObservedAt + out.TerminalAt = t.Projection.TerminalAt + if t.Projection.PublicHandle != nil && *t.Projection.PublicHandle != "" { + out.PublicHandle = boundedText(*t.Projection.PublicHandle, maxHistoryIdentifier) + } + if out.Title == "" || t.Reason == model.ReasonFirstAuthoritativeState || t.Projection.PublicHandle != nil { + out.Title = episodeTitle(out) + } + if out.RecurrenceCount < t.Journal.RecurrenceCount { + out.RecurrenceCount = t.Journal.RecurrenceCount + } + + if t.Reason != model.ReasonOperatorArtifactRecorded { + if out.InitialPublicationReason == "" { + out.InitialPublicationReason = string(t.Reason) + } + } + out.LatestMaterialReason = string(t.Reason) + + if concl := t.Projection.Assessment; concl != nil { + if concl.SufficientReasonSummary != "" { + out.EvidenceConclusion = boundedText(concl.SufficientReasonSummary, maxHistoryDetail) + } + out.ImpactSummary = impactSummary(concl.Impact) + } + + switch t.JournalKind { //nolint:exhaustive // only the accumulating journal kinds contribute to the two bounded summary lists; every other kind updates the scalar fields above. + case model.JournalInvestigationStarted: + out.InvestigationStarted = true + out.InvestigationWork = appendBounded(out.InvestigationWork, investigationEntry(t), true) + case model.JournalInvestigationChanged, model.JournalEvidenceConclusion: + out.InvestigationWork = appendBounded(out.InvestigationWork, investigationEntry(t), true) + case model.JournalOperatorNote, model.JournalCapturedVerdict: + // Never collapsed: two artifacts that happen to read alike are still + // two separate durable operator records. + out.RecordedOperatorContext = appendBounded(out.RecordedOperatorContext, operatorContextEntry(t), false) + } + + if out.TerminalAt != nil { + seconds := int64(out.TerminalAt.Sub(out.EffectiveStartedAt) / time.Second) + if seconds < 0 { + seconds = 0 + } + out.DurationSeconds = &seconds + out.FinalOutcome = finalOutcome(t, out) + out.RemainingUncertainty = remainingUncertainty(t) + } + + if err := out.Validate(); err != nil { + return model.EpisodeSummary{}, fmt.Errorf("situation: project episode: %w", err) + } + return out, nil +} + +// validateFold rejects every incoherent fold: a Transition from another +// Situation, one that skips or repeats a sequence, one that moves time +// backwards, and any fold at all onto a terminal Episode — a later firing +// creates a separately linked Situation through Plan 1/2 recurrence +// ownership. +func validateFold(prior model.EpisodeSummary, t model.Transition) error { + switch { + case prior.SituationID != t.SituationID: + return fmt.Errorf("situation: project episode: transition belongs to situation %q, summary to %q", + t.SituationID, prior.SituationID) + case prior.TerminalAt != nil: + return fmt.Errorf("situation: project episode: episode is terminal at %s and never reopens", prior.TerminalAt) + case t.Sequence != prior.SourceTransitionSequence+1: + return fmt.Errorf("situation: project episode: transition sequence %d is not contiguous with summary sequence %d", + t.Sequence, prior.SourceTransitionSequence) + case t.CreatedAt.Before(prior.UpdatedAt): + return fmt.Errorf("situation: project episode: transition created at %s precedes summary updated at %s", + t.CreatedAt, prior.UpdatedAt) + } + return nil +} + +// episodeTitle names the Episode by the immutable Situation handle once one +// is assigned, and by the Situation ID before that. R3 restricts the fold +// to closed codes and the bounded Sufficient-reason summary, so there is no +// human-authored title to copy; renderers compose the operator-facing +// heading from this label plus the summary's own fields. +func episodeTitle(s model.EpisodeSummary) string { + if s.PublicHandle != "" { + return boundedText("Situation "+s.PublicHandle, maxHistoryHeadline) + } + return boundedText("Situation "+s.SituationID, maxHistoryHeadline) +} + +func investigationEntry(t model.Transition) string { + entry := t.Journal.Headline + if t.Journal.Detail != "" { + entry += " — " + t.Journal.Detail + } + return boundedText(entry, maxHistoryDetail) +} + +func operatorContextEntry(t model.Transition) string { + entry := t.Journal.Headline + if t.Journal.AttributedActor != "" { + entry = t.Journal.AttributedActor + ": " + entry + } + return boundedText(entry, maxHistoryDetail) +} + +// appendBounded keeps a bounded accumulation: the list never exceeds its +// bound (the oldest entry is dropped; the immutable Transition ledger keeps +// the full history). With collapseRepeat, an entry identical to the last +// one does not grow the list — repeated identical investigation lines say +// nothing new. +func appendBounded(list []string, entry string, collapseRepeat bool) []string { + if strings.TrimSpace(entry) == "" { + return list + } + if collapseRepeat && len(list) > 0 && list[len(list)-1] == entry { + return list + } + list = append(list, entry) + if len(list) > maxHistoryListEntries { + list = list[len(list)-maxHistoryListEntries:] + } + return list +} + +func impactSummary(i model.Impact) string { + switch i { //nolint:exhaustive // unknown (and any zero value) is the default. + case model.ImpactConfirmed: + return "Confirmed impact" + case model.ImpactSuspected: + return "Suspected impact" + case model.ImpactNoneObserved: + return "No impact observed" + default: + return "Impact unknown" + } +} + +// finalOutcome states what happened, and never collapses a terminal +// Situation to "no action": with no recorded operator artifact it says so +// explicitly rather than implying AlertINT or anyone else acted. +func finalOutcome(t model.Transition, s model.EpisodeSummary) string { + if t.Projection.TerminalReason != nil { + return boundedText("Closed with uncertainty ("+string(*t.Projection.TerminalReason)+")", maxHistoryDetail) + } + if len(s.RecordedOperatorContext) == 0 { + return "Recovered without recorded operator intervention" + } + return "Recovered with recorded operator context" +} + +func remainingUncertainty(t model.Transition) string { + if t.Projection.TerminalReason == nil { + return "" + } + out := "AlertINT could not confirm resolution: " + string(*t.Projection.TerminalReason) + "." + if c := t.Projection.Assessment; c != nil && c.EvidenceQuality != model.EvidenceQualityComplete { + out += " Evidence quality: " + string(c.EvidenceQuality) + "." + } + return boundedText(out, maxHistoryDetail) +} + +func attentionRank(a model.Attention) int { + switch a { //nolint:exhaustive // observe (and any zero value) is rank 0, the default. + case model.AttentionUrgent: + return 2 + case model.AttentionInvestigate: + return 1 + default: + return 0 + } +} + +// ---------------------------------------------------------------------- +// Orientation +// ---------------------------------------------------------------------- + +// Orientation is the single currently-emphasized phase of the Situation's +// compact orientation line. It is pure derived presentation, never another +// persisted lifecycle: phase movement alone creates no Transition. +type Orientation string + +const ( + OrientationObserved Orientation = "observed" + OrientationInvestigating Orientation = "investigating" + OrientationMonitoring Orientation = "monitoring" + OrientationRecovered Orientation = "recovered" + OrientationClosedUncertain Orientation = "closed_uncertain" +) + +// DeriveOrientation reports which phase the root currently emphasizes, +// from the accepted Transition and the folded Episode summary alone: +// +// Observed -> Investigating -> Monitoring -> Recovered +// Observed -> Investigating -> Closed uncertain +// +// A refire returns emphasis from Monitoring to Investigating (the summary +// keeps InvestigationStarted), and a direct closed_unknown never invents +// Monitoring. Which phases the rendered chain SHOWS — in particular whether +// a closed_unknown chain includes Monitoring at all — additionally needs +// the Situation's Transition history, which the Slack renderer reads. +func DeriveOrientation(summary model.EpisodeSummary, latest model.Transition) Orientation { + switch latest.Lifecycle { //nolint:exhaustive // active is the default branch, where Observed and Investigating are told apart. + case model.LifecycleRecovered: + return OrientationRecovered + case model.LifecycleClosedUnknown: + return OrientationClosedUncertain + case model.LifecycleRecoveryPending: + return OrientationMonitoring + default: + if summary.InvestigationStarted || investigationCurrent(latest.ActionContract) || + latest.ActionContract.OperatorActionRequired != nil { + return OrientationInvestigating + } + return OrientationObserved + } +} + +// ---------------------------------------------------------------------- +// Composition +// ---------------------------------------------------------------------- + +// BuildHistoryCommit runs this task's three pure functions in the order the +// fenced commit applies them: derive the Transitions, fold the Episode +// summary once per Transition in sequence order, then plan the notification +// intents for the resulting publication. +// +// publication supplies only the delivery-side context — the committed +// contract deadline, whether the root is published, the last delivered root +// promise, the last main-channel poke, the operator's Slack floor, and the +// repage cooldown. Its Transitions and Summary fields are derived here and +// must be left zero; Situation/Drill/Now must match change. +// +// Task 5 may equally call BuildTransitions, ProjectEpisode, and +// PlanNotificationIntents itself when it needs the intermediate values. +func BuildHistoryCommit(change AuthoritativeChange, publication PublicationInput) (HistoryCommit, error) { + if len(publication.Transitions) != 0 || publication.Summary.SituationID != "" || publication.Summary.Version != 0 { + return HistoryCommit{}, errors.New("situation: build history commit: publication input must not carry transitions or a summary; both are derived here") + } + if publication.Situation.ID != change.Situation.ID { + return HistoryCommit{}, fmt.Errorf("situation: build history commit: publication situation %q does not match change situation %q", + publication.Situation.ID, change.Situation.ID) + } + if !publication.Now.Equal(change.Now) { + return HistoryCommit{}, fmt.Errorf("situation: build history commit: publication now %s does not match change now %s", + publication.Now, change.Now) + } + if publication.Drill != change.Drill { + return HistoryCommit{}, errors.New("situation: build history commit: publication and change disagree about the Drill marker") + } + + transitions, err := BuildTransitions(change) + if err != nil { + return HistoryCommit{}, err + } + + commit := HistoryCommit{Transitions: transitions} + prior := change.PriorSummary + for i, tr := range transitions { + folded, err := ProjectEpisode(prior, tr) + if err != nil { + return HistoryCommit{}, fmt.Errorf("situation: build history commit: transition %d: %w", i, err) + } + commit.Summary = &folded + prior = commit.Summary + } + + publication.Transitions = transitions + if commit.Summary != nil { + publication.Summary = *commit.Summary + } else if change.PriorSummary != nil { + publication.Summary = *change.PriorSummary + } + if publication.PriorTransition == nil { + publication.PriorTransition = change.PriorTransition + } + intents, err := PlanNotificationIntents(publication) + if err != nil { + return HistoryCommit{}, err + } + commit.Intents = intents + return commit, nil +} + +// ---------------------------------------------------------------------- +// Small local helpers. +// ---------------------------------------------------------------------- + +// boundedText truncates s to at most limit bytes without splitting a rune. +func boundedText(s string, limit int) string { + if len(s) <= limit { + return s + } + cut := s[:limit] + for len(cut) > 0 && !utf8.ValidString(cut) { + cut = cut[:len(cut)-1] + } + return cut +} + +func derefAlertINTAction(a *model.AlertINTAction) string { + if a == nil { + return "" + } + return string(*a) +} + +func derefAlertINTStatus(s *model.AlertINTStatus) string { + if s == nil { + return "" + } + return string(*s) +} + +func derefOperatorAction(o *model.OperatorAction) string { + if o == nil { + return "" + } + return string(*o) +} + +func derefWaitReason(w *model.WaitReason) string { + if w == nil { + return "" + } + return string(*w) +} diff --git a/internal/situation/history_test.go b/internal/situation/history_test.go new file mode 100644 index 0000000..7083a8f --- /dev/null +++ b/internal/situation/history_test.go @@ -0,0 +1,1426 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "strings" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 4 fixtures. Every helper here is prefixed `hs` (history +// slice) so it never collides with Plan 2's existing package-level test +// helpers (mustTime, fixedClock, baseSnapshotInput, ct*, ...). mustTime +// itself is reused from snapshot_test.go. +// ---------------------------------------------------------------------- + +const ( + hsSituationID = "1f0f5a0c-0000-4000-8000-000000000001" + hsGroupKey = "service=checkout" + hsIncidentID = "incident-0001" + hsHash1 = "sha256:1111111111" + hsHash2 = "sha256:2222222222" +) + +func hsNow(t *testing.T) time.Time { + t.Helper() + return mustTime(t, "2026-09-06T10:00:00Z") +} + +// hsRunningTriageContract is a valid nonterminal Operator contract in which +// AlertINT is currently running Acute Triage. +func hsRunningTriageContract(next time.Time) model.ActionContract { + action := model.AlertINTActionRunAcuteTriage + status := model.AlertINTStatusRunning + return model.ActionContract{ + NextActor: model.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: timePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnTriageOutcome}, + } +} + +// hsMonitoringContract is a valid nonterminal Operator contract in which no +// AlertINT investigation is current — AlertINT is only monitoring. +func hsMonitoringContract(next time.Time) model.ActionContract { + action := model.AlertINTActionMonitorSituation + status := model.AlertINTStatusWaiting + wait := model.WaitReasonSourceChange + return model.ActionContract{ + NextActor: model.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: timePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnMaterialInput}, + WaitReason: &wait, + } +} + +// hsOperatorContract is a valid nonterminal Operator contract that hands the +// next move to a human while AlertINT's own Acute Triage work continues — +// the contract validator's own rule that "operator action wins even when +// AlertINT work continues". Keeping the AlertINT action current isolates +// the handoff from an investigation conclusion. +func hsOperatorContract(next time.Time) model.ActionContract { + op := model.OperatorActionInvestigateSituation + action := model.AlertINTActionRunAcuteTriage + status := model.AlertINTStatusRunning + return model.ActionContract{ + NextActor: model.NextActorOperator, + AlertINTAction: &action, + AlertINTStatus: &status, + OperatorActionRequired: &op, + NextUpdateAt: timePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnMaterialInput}, + } +} + +// hsTerminalContract is the only shape a terminal Transition may carry. +func hsTerminalContract() model.ActionContract { + return model.ActionContract{NextActor: model.NextActorNone} +} + +func hsConclusion() model.AssessmentConclusion { + return model.AssessmentConclusion{ + Persistence: model.PersistenceSustained, + Impact: model.ImpactSuspected, + Novelty: model.NoveltyNew, + Causality: model.CausalityCorrelated, + EvidenceQuality: model.EvidenceQualityComplete, + LimitationCodes: []string{}, + SufficientReasonCode: reasonCodeCriticalAnchor, + SufficientReasonSummary: "Confirmed active critical source severity.", + } +} + +func hsAssessment(contract model.ActionContract, concl model.AssessmentConclusion, + lifecycle model.Lifecycle, attention model.Attention) model.Assessment { + a := model.Assessment{ + SchemaVersion: model.AssessmentSchemaVersion, + Persistence: concl.Persistence, + Impact: concl.Impact, + Novelty: concl.Novelty, + Causality: concl.Causality, + Attention: attention, + Lifecycle: lifecycle, + EvidenceQuality: concl.EvidenceQuality, + ActionContract: contract, + Limitations: []model.Limitation{}, + Cadence: model.CadenceFast, + } + if lifecycle.Terminal() { + a.Cadence = model.Cadence("") + } + if concl.SufficientReasonCode != "" { + a.SufficientReason = &model.SufficientReason{ + Code: concl.SufficientReasonCode, + CandidateID: "candidate-" + concl.SufficientReasonCode, + Summary: concl.SufficientReasonSummary, + EvidenceRefs: []string{"fact-anchor"}, + } + } + return a +} + +func hsSituation(now time.Time, inputVersion int, lifecycle model.Lifecycle, attention model.Attention) model.Situation { + return model.Situation{ + ID: hsSituationID, + GroupKey: hsGroupKey, + Lifecycle: lifecycle, + Attention: attention, + InputVersion: inputVersion, + OpenedAt: now.Add(-time.Hour), + EffectiveStartedAt: now.Add(-90 * time.Minute), + EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload, + FirstReceivedAt: now.Add(-89 * time.Minute), + LastLifecycleObservedAt: now, + NextAssessmentAt: now.Add(time.Minute), + DueReasons: []model.DueReason{model.DueMembershipChanged}, + CreatedAt: now.Add(-time.Hour), + UpdatedAt: now, + } +} + +// hsChange is the baseline AuthoritativeChange: an active, investigate-level +// Situation whose current authoritative Assessment has AlertINT running +// Acute Triage, with no prior Transition (a first authoritative state). +func hsChange(t *testing.T) AuthoritativeChange { + t.Helper() + now := hsNow(t) + concl := hsConclusion() + contract := hsRunningTriageContract(now.Add(time.Minute)) + sit := hsSituation(now, 7, model.LifecycleActive, model.AttentionInvestigate) + return AuthoritativeChange{ + Situation: sit, + AssessmentID: stringPtrOf("assessment-0001"), + Assessment: hsAssessment(contract, concl, model.LifecycleActive, model.AttentionInvestigate), + Derivation: model.DerivationModelValidated, + Projection: model.ProjectionFacts{ + EffectiveStartedAt: sit.EffectiveStartedAt, + EffectiveStartedAtBasis: sit.EffectiveStartedAtBasis, + Assessment: &concl, + }, + MaterialFactHash: hsHash1, + EvidenceRefs: []string{"fact-b", "fact-a"}, + Incidents: []IncidentState{hsIncident("pending", nil)}, + RecurrenceCount: 0, + Now: now, + } +} + +func hsIncident(phase string, decision *string) IncidentState { + return IncidentState{ + ID: hsIncidentID, + GroupKey: hsGroupKey, + Status: "ready", + Triage: TriageState{Phase: phase, Decision: decision}, + } +} + +// hsFirst builds the single Transition the baseline change produces, for use +// as the next cycle's PriorTransition. +func hsFirst(t *testing.T) model.Transition { + t.Helper() + got, err := BuildTransitions(hsChange(t)) + if err != nil { + t.Fatalf("BuildTransitions(baseline): %v", err) + } + if len(got) != 1 { + t.Fatalf("BuildTransitions(baseline) = %d transitions, want 1", len(got)) + } + return got[0] +} + +// hsPriorSummary folds the baseline Transition into the first Episode +// summary, for use as the next cycle's PriorSummary. +func hsPriorSummary(t *testing.T, prior model.Transition) model.EpisodeSummary { + t.Helper() + sum, err := ProjectEpisode(nil, prior) + if err != nil { + t.Fatalf("ProjectEpisode(nil, first): %v", err) + } + return sum +} + +// hsNext returns a second-cycle change carrying the prior Transition and +// summary at a bumped input version, one minute later. +func hsNext(t *testing.T) AuthoritativeChange { + t.Helper() + prior := hsFirst(t) + sum := hsPriorSummary(t, prior) + next := hsChange(t) + next.Now = next.Now.Add(time.Minute) + next.Situation.InputVersion = 8 + next.Situation.UpdatedAt = next.Now + next.PriorTransition = &prior + next.PriorSummary = &sum + next.AssessmentID = stringPtrOf("assessment-0002") + next.Assessment.ActionContract = hsRunningTriageContract(next.Now.Add(time.Minute)) + return next +} + +func hsArtifact(id, kind string, occurredAt time.Time) OperatorArtifactInput { + a := OperatorArtifactInput{ + InputID: id, + Kind: kind, + AppliedInputVersion: 8, + OccurredAt: occurredAt, + AttributedActor: "operator@example.com", + Headline: "Checked the deploy log", + Detail: "Rollback started at 09:58.", + } + switch kind { + case artifactKindAnnotation: + a.AnnotationID = stringPtrOf("annotation-" + id) + case artifactKindVerdict: + a.VerdictID = stringPtrOf("verdict-" + id) + } + return a +} + +// hsUseReason rewrites both halves of the Sufficient reason a change +// carries — the projection's closed conclusion code (what materiality and +// priority read) and the Assessment's own selection. +func hsUseReason(c *AuthoritativeChange, code string) { + concl := hsConclusion() + if c.Projection.Assessment != nil { + concl = *c.Projection.Assessment + } + concl.SufficientReasonCode = code + concl.SufficientReasonSummary = "Sufficient reason " + code + "." + c.Projection.Assessment = &concl + if c.Assessment.SufficientReason != nil { + c.Assessment.SufficientReason.Code = code + c.Assessment.SufficientReason.Summary = concl.SufficientReasonSummary + } +} + +// hsRecovered turns a change into a committed recovery: terminal lifecycle, +// terminal contract, recovery observation retained, no terminal reason +// (migration 0014: recovered carries terminal_at with a NULL terminal_reason). +func hsRecovered(c *AuthoritativeChange) { + c.Situation.Lifecycle = model.LifecycleRecovered + c.Assessment.Lifecycle = model.LifecycleRecovered + c.Assessment.Cadence = model.Cadence("") + c.Assessment.ActionContract = hsTerminalContract() + c.Situation.RecoveryObservedAt = timePtr(c.Now.Add(-10 * time.Minute)) + c.Situation.TerminalAt = timePtr(c.Now) + c.Projection.RecoveryObservedAt = timePtr(c.Now.Add(-10 * time.Minute)) + c.Projection.TerminalAt = timePtr(c.Now) +} + +// hsClosedUnknown turns a change into a committed closure with uncertainty. +func hsClosedUnknown(c *AuthoritativeChange) { + c.Situation.Lifecycle = model.LifecycleClosedUnknown + c.Assessment.Lifecycle = model.LifecycleClosedUnknown + c.Assessment.Cadence = model.Cadence("") + c.Assessment.ActionContract = hsTerminalContract() + reason := model.TerminalReasonObservationDeadline + c.Situation.TerminalAt = timePtr(c.Now) + c.Situation.TerminalReason = &reason + c.Projection.TerminalAt = timePtr(c.Now) + c.Projection.TerminalReason = &reason +} + +func hsOnly(t *testing.T, change AuthoritativeChange) model.Transition { + t.Helper() + got, err := BuildTransitions(change) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 1 { + t.Fatalf("BuildTransitions = %d transitions, want exactly 1: %+v", len(got), got) + } + if err := got[0].Validate(); err != nil { + t.Fatalf("derived transition failed model validation: %v", err) + } + return got[0] +} + +// ---------------------------------------------------------------------- +// Step 1: the material-change catalog. +// ---------------------------------------------------------------------- + +func TestBuildTransitionsCatalog(t *testing.T) { + now := hsNow(t) + + cases := []struct { + name string + change func(t *testing.T) AuthoritativeChange + reason model.TransitionReason + actor model.TransitionActor + journal model.JournalKind + evidence []string + pokeAllowed bool + headlineHas string + }{ + { + name: "first authoritative state", + change: hsChange, + reason: model.ReasonFirstAuthoritativeState, + actor: model.ActorLLM, + journal: model.JournalPublication, + evidence: []string{"fact-a", "fact-anchor", "fact-b"}, + pokeAllowed: true, + }, + { + name: "material assessment change", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + concl := hsConclusion() + concl.Impact = model.ImpactConfirmed + c.Projection.Assessment = &concl + c.Assessment.Impact = model.ImpactConfirmed + return c + }, + reason: model.ReasonMaterialAssessmentChanged, + actor: model.ActorLLM, + journal: model.JournalInvestigationChanged, + evidence: []string{"fact-a", "fact-anchor", "fact-b"}, + pokeAllowed: false, + }, + { + name: "attention change", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + return c + }, + reason: model.ReasonAttentionChanged, + actor: model.ActorLLM, + journal: model.JournalOperatorContractChanged, + pokeAllowed: true, + headlineHas: "urgent", + }, + { + name: "operator contract change", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + return c + }, + reason: model.ReasonOperatorContractChanged, + actor: model.ActorDeterministicController, + journal: model.JournalOperatorContractChanged, + pokeAllowed: true, + headlineHas: "operator", + }, + { + name: "investigation started", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsChange(t) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + prior, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions(prior): %v", err) + } + sum := hsPriorSummary(t, prior[0]) + n := hsNext(t) + n.PriorTransition = &prior[0] + n.PriorSummary = &sum + return n + }, + reason: model.ReasonInvestigationStarted, + actor: model.ActorDeterministicController, + journal: model.JournalInvestigationStarted, + pokeAllowed: false, + }, + { + name: "investigation concluded", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + return c + }, + reason: model.ReasonInvestigationConcluded, + actor: model.ActorLLM, + journal: model.JournalEvidenceConclusion, + pokeAllowed: false, + }, + { + name: "recovery observed", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.Situation.Lifecycle = model.LifecycleRecoveryPending + c.Assessment.Lifecycle = model.LifecycleRecoveryPending + c.Situation.RecoveryObservedAt = timePtr(c.Now) + c.Projection.RecoveryObservedAt = timePtr(c.Now) + c.Projection.GraceUntil = timePtr(c.Now.Add(10 * time.Minute)) + return c + }, + reason: model.ReasonRecoveryObserved, + actor: model.ActorDeterministicController, + journal: model.JournalRecoveryPending, + pokeAllowed: false, + }, + { + name: "recovery failed", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsChange(t) + c.Situation.Lifecycle = model.LifecycleRecoveryPending + c.Assessment.Lifecycle = model.LifecycleRecoveryPending + c.Projection.RecoveryObservedAt = timePtr(c.Now.Add(-time.Minute)) + prior, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions(prior): %v", err) + } + sum := hsPriorSummary(t, prior[0]) + n := hsNext(t) + n.PriorTransition = &prior[0] + n.PriorSummary = &sum + return n + }, + reason: model.ReasonRecoveryFailed, + actor: model.ActorDeterministicController, + journal: model.JournalRecoveryRefired, + pokeAllowed: false, + }, + { + name: "recovered", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + hsRecovered(&c) + return c + }, + reason: model.ReasonRecovered, + actor: model.ActorDeterministicController, + journal: model.JournalRecovered, + pokeAllowed: false, + }, + { + name: "closed unknown", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + hsClosedUnknown(&c) + return c + }, + reason: model.ReasonClosedUnknown, + actor: model.ActorDeterministicController, + journal: model.JournalClosedUnknown, + pokeAllowed: false, + headlineHas: "uncertain", + }, + { + name: "recurrence milestone", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.RecurrenceCount = 5 + return c + }, + reason: model.ReasonRecurrenceMilestone, + actor: model.ActorDeterministicController, + journal: model.JournalRecurrenceMilestone, + pokeAllowed: false, + headlineHas: "5", + }, + { + name: "triage state changed by a B+ skip decision", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.TriageDecisions = []TriageDecision{{ + IncidentID: hsIncidentID, + Decision: TriageDecisionSkip, + DecisionReason: DecisionReasonCleanSkip, + SituationID: hsSituationID, + SituationInputVersion: 8, + DecidedAt: c.Now, + }} + return c + }, + reason: model.ReasonTriageStateChanged, + actor: model.ActorDeterministicController, + journal: model.JournalInvestigationChanged, + pokeAllowed: false, + headlineHas: "skipped", + }, + { + name: "triage state changed by the pre-claim minimum-member clean skip", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.Situation.DueReasons = []model.DueReason{model.DueTriageChanged} + c.Incidents = []IncidentState{hsIncident(triagePhaseSkipped, nil)} + return c + }, + reason: model.ReasonTriageStateChanged, + actor: model.ActorDeterministicController, + journal: model.JournalInvestigationChanged, + pokeAllowed: false, + headlineHas: "skipped", + }, + { + name: "attributed annotation", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, now)} + return c + }, + reason: model.ReasonOperatorArtifactRecorded, + actor: model.ActorAttributedOperator, + journal: model.JournalOperatorNote, + evidence: []string{"annotation:annotation-input-1"}, + pokeAllowed: false, + headlineHas: "deploy log", + }, + { + name: "captured verdict", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-2", artifactKindVerdict, now)} + return c + }, + reason: model.ReasonOperatorArtifactRecorded, + actor: model.ActorAttributedOperator, + journal: model.JournalCapturedVerdict, + evidence: []string{"verdict:verdict-input-2"}, + pokeAllowed: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := hsOnly(t, tc.change(t)) + if got.Reason != tc.reason { + t.Errorf("reason = %q, want %q", got.Reason, tc.reason) + } + if got.Actor != tc.actor { + t.Errorf("actor = %q, want %q", got.Actor, tc.actor) + } + if got.JournalKind != tc.journal { + t.Errorf("journal kind = %q, want %q", got.JournalKind, tc.journal) + } + if tc.evidence != nil { + if strings.Join(got.EvidenceRefs, ",") != strings.Join(tc.evidence, ",") { + t.Errorf("evidence refs = %v, want %v", got.EvidenceRefs, tc.evidence) + } + } + if pokeEligible := got.InterruptionPriority != nil; pokeEligible != tc.pokeAllowed { + t.Errorf("main-channel poke eligible = %v (priority %v), want %v", + pokeEligible, got.InterruptionPriority, tc.pokeAllowed) + } + if tc.headlineHas != "" { + full := got.Journal.Headline + " " + got.Journal.Detail + if !strings.Contains(strings.ToLower(full), strings.ToLower(tc.headlineHas)) { + t.Errorf("journal %q / %q contains no %q", got.Journal.Headline, got.Journal.Detail, tc.headlineHas) + } + } + if got.Journal.OccurredAt.IsZero() { + t.Error("journal occurred_at is zero") + } + if got.Projection.EffectiveStartedAt.IsZero() { + t.Error("projection effective_started_at is zero (R3)") + } + }) + } +} + +// TestBuildTransitionsCleanSkipNeverStartsInvestigation pins the brief's +// explicit clean-skip rule: the skip journals, but never reads as +// investigation having run. +func TestBuildTransitionsCleanSkipNeverStartsInvestigation(t *testing.T) { + c := hsNext(t) + c.Situation.DueReasons = []model.DueReason{model.DueTriageChanged} + c.Incidents = []IncidentState{hsIncident(triagePhaseSkipped, nil)} + tr := hsOnly(t, c) + + sum, err := ProjectEpisode(c.PriorSummary, tr) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if sum.InvestigationStarted { + t.Error("a clean skip must never set InvestigationStarted") + } + if tr.JournalKind == model.JournalInvestigationStarted { + t.Error("a clean skip must never journal investigation_started") + } +} + +// ---------------------------------------------------------------------- +// Step 1: R1 — artifact ordering. +// ---------------------------------------------------------------------- + +func TestBuildTransitionsArtifactOrdering(t *testing.T) { + now := hsNow(t) + + t.Run("two artifacts plus a material state change yield three in order", func(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{ + hsArtifact("input-1", artifactKindAnnotation, now), + hsArtifact("input-2", artifactKindVerdict, now.Add(time.Second)), + } + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 3 { + t.Fatalf("got %d transitions, want 3", len(got)) + } + wantReasons := []model.TransitionReason{ + model.ReasonOperatorArtifactRecorded, + model.ReasonOperatorArtifactRecorded, + model.ReasonAttentionChanged, + } + for i, want := range wantReasons { + if got[i].Reason != want { + t.Errorf("transition %d reason = %q, want %q", i, got[i].Reason, want) + } + if err := got[i].Validate(); err != nil { + t.Errorf("transition %d failed model validation: %v", i, err) + } + } + if got[0].OperatorArtifactInputID == nil || *got[0].OperatorArtifactInputID != "input-1" { + t.Errorf("first artifact transition input id = %v, want input-1", got[0].OperatorArtifactInputID) + } + if got[1].OperatorArtifactInputID == nil || *got[1].OperatorArtifactInputID != "input-2" { + t.Errorf("second artifact transition input id = %v, want input-2", got[1].OperatorArtifactInputID) + } + if got[2].OperatorArtifactInputID != nil { + t.Error("controller-state transition must not carry an artifact input id") + } + for i := range got { + if want := i + 2; got[i].Sequence != want { + t.Errorf("transition %d sequence = %d, want %d", i, got[i].Sequence, want) + } + } + }) + + t.Run("two artifacts with no state change yield two", func(t *testing.T) { + c := hsNext(t) + c.OperatorArtifacts = []OperatorArtifactInput{ + hsArtifact("input-1", artifactKindAnnotation, now), + hsArtifact("input-2", artifactKindAnnotation, now.Add(time.Second)), + } + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d transitions, want 2", len(got)) + } + for i := range got { + if got[i].Reason != model.ReasonOperatorArtifactRecorded { + t.Errorf("transition %d reason = %q, want operator_artifact_recorded", i, got[i].Reason) + } + } + }) + + t.Run("a terminal state change journals the pending artifact first", func(t *testing.T) { + c := hsNext(t) + hsClosedUnknown(&c) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-9", artifactKindAnnotation, now)} + + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d transitions, want 2", len(got)) + } + if got[0].Reason != model.ReasonOperatorArtifactRecorded { + t.Errorf("first reason = %q, want operator_artifact_recorded", got[0].Reason) + } + if got[1].Reason != model.ReasonClosedUnknown { + t.Errorf("second reason = %q, want closed_unknown", got[1].Reason) + } + if got[0].Sequence >= got[1].Sequence { + t.Errorf("artifact sequence %d must precede terminal sequence %d", got[0].Sequence, got[1].Sequence) + } + }) + + t.Run("nothing pending and nothing material yields no transitions", func(t *testing.T) { + got, err := BuildTransitions(hsNext(t)) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 0 { + t.Fatalf("got %d transitions, want 0: %+v", len(got), got) + } + }) +} + +// ---------------------------------------------------------------------- +// Step 1: R4 — materiality is semantic. +// ---------------------------------------------------------------------- + +func TestMaterialityRevalidatedReuseCreatesNoTransition(t *testing.T) { + c := hsNext(t) + // Exactly Plan 2's revalidated_reuse shape: a new authoritative + // Assessment ID, a new Sufficient-reason candidate ID, new per-input + // evidence identities, a freshly recomputed next_update_at — and an + // unchanged semantic tuple. + c.Derivation = model.DerivationRevalidatedReuse + c.AssessmentID = stringPtrOf("assessment-0002") + c.Assessment.SufficientReason.CandidateID = "candidate-refreshed" + c.Assessment.SufficientReason.EvidenceRefs = []string{"fact-anchor-v8"} + c.Assessment.ActionContract.NextUpdateAt = timePtr(c.Now.Add(17 * time.Minute)) + c.EvidenceRefs = []string{"fact-a-v8", "fact-b-v8"} + c.MaterialFactHash = hsHash2 + + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 0 { + t.Fatalf("revalidated_reuse produced %d transitions, want 0: %+v", len(got), got) + } +} + +func TestMaterialityTupleMembers(t *testing.T) { + cases := []struct { + name string + muton func(c *AuthoritativeChange) + want model.TransitionReason + }{ + {"lifecycle", func(c *AuthoritativeChange) { + c.Situation.Lifecycle = model.LifecycleRecoveryPending + c.Assessment.Lifecycle = model.LifecycleRecoveryPending + c.Situation.RecoveryObservedAt = timePtr(c.Now) + c.Projection.RecoveryObservedAt = timePtr(c.Now) + }, model.ReasonRecoveryObserved}, + {"attention", func(c *AuthoritativeChange) { + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + }, model.ReasonAttentionChanged}, + {"next actor and operator action", func(c *AuthoritativeChange) { + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + }, model.ReasonOperatorContractChanged}, + {"alertint status", func(c *AuthoritativeChange) { + blocked := model.AlertINTStatusBlocked + c.Assessment.ActionContract.AlertINTStatus = &blocked + }, model.ReasonOperatorContractChanged}, + {"next update on", func(c *AuthoritativeChange) { + c.Assessment.ActionContract.NextUpdateOn = []model.NextUpdateOn{model.NextUpdateOnSourceResolution} + }, model.ReasonOperatorContractChanged}, + {"wait reason", func(c *AuthoritativeChange) { + w := model.WaitReasonAcuteTriageBackoff + c.Assessment.ActionContract.WaitReason = &w + }, model.ReasonOperatorContractChanged}, + {"sufficient reason code", func(c *AuthoritativeChange) { + concl := hsConclusion() + concl.SufficientReasonCode = reasonCodeDurationOutlier + c.Projection.Assessment = &concl + c.Assessment.SufficientReason.Code = reasonCodeDurationOutlier + }, model.ReasonMaterialAssessmentChanged}, + {"assessment conclusion code", func(c *AuthoritativeChange) { + concl := hsConclusion() + concl.Causality = model.CausalitySupported + c.Projection.Assessment = &concl + c.Assessment.Causality = model.CausalitySupported + }, model.ReasonMaterialAssessmentChanged}, + {"limitation codes", func(c *AuthoritativeChange) { + concl := hsConclusion() + concl.LimitationCodes = []string{"semantic_assessment_unavailable"} + c.Projection.Assessment = &concl + }, model.ReasonMaterialAssessmentChanged}, + {"triage decision", func(c *AuthoritativeChange) { + c.TriageDecisions = []TriageDecision{{ + IncidentID: hsIncidentID, Decision: TriageDecisionSkip, + DecisionReason: DecisionReasonCleanSkip, SituationID: hsSituationID, + SituationInputVersion: 8, DecidedAt: c.Now, + }} + }, model.ReasonTriageStateChanged}, + {"recurrence milestone", func(c *AuthoritativeChange) { + c.RecurrenceCount = 10 + }, model.ReasonRecurrenceMilestone}, + } + + for _, tc := range cases { + t.Run(tc.name+" is material", func(t *testing.T) { + c := hsNext(t) + tc.muton(&c) + got := hsOnly(t, c) + if got.Reason != tc.want { + t.Errorf("reason = %q, want %q", got.Reason, tc.want) + } + }) + } +} + +func TestMaterialityIgnoresNonSemanticChurn(t *testing.T) { + cases := []struct { + name string + muton func(c *AuthoritativeChange) + }{ + {"assessment id", func(c *AuthoritativeChange) { c.AssessmentID = stringPtrOf("assessment-9999") }}, + {"material fact hash", func(c *AuthoritativeChange) { c.MaterialFactHash = hsHash2 }}, + {"next_update_at", func(c *AuthoritativeChange) { + c.Assessment.ActionContract.NextUpdateAt = timePtr(c.Now.Add(42 * time.Minute)) + }}, + {"reason candidate id", func(c *AuthoritativeChange) { + c.Assessment.SufficientReason.CandidateID = "candidate-v8" + }}, + {"evidence ids", func(c *AuthoritativeChange) { c.EvidenceRefs = []string{"fact-x", "fact-y"} }}, + {"sufficient reason prose", func(c *AuthoritativeChange) { + concl := hsConclusion() + concl.SufficientReasonSummary = "Reworded but identical judgment." + c.Projection.Assessment = &concl + }}, + {"input version alone", func(c *AuthoritativeChange) { c.Situation.InputVersion = 99 }}, + {"non-milestone recurrence count", func(c *AuthoritativeChange) { c.RecurrenceCount = 2 }}, + } + + for _, tc := range cases { + t.Run(tc.name+" is not material", func(t *testing.T) { + c := hsNext(t) + tc.muton(&c) + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 0 { + t.Fatalf("got %d transitions, want 0: %+v", len(got), got) + } + }) + } +} + +// ---------------------------------------------------------------------- +// Step 2: deterministic identity and authority. +// ---------------------------------------------------------------------- + +func TestTransitionIdentityIsDeterministic(t *testing.T) { + build := func(mut func(c *AuthoritativeChange)) []model.Transition { + t.Helper() + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + if mut != nil { + mut(&c) + } + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + return got + } + + a := build(nil) + b := build(nil) + if len(a) != 2 || len(b) != 2 { + t.Fatalf("want 2 transitions per run, got %d and %d", len(a), len(b)) + } + for i := range a { + if a[i].ID != b[i].ID { + t.Errorf("transition %d id is not deterministic: %q vs %q", i, a[i].ID, b[i].ID) + } + if a[i].ID == "" { + t.Errorf("transition %d id is empty", i) + } + } + if a[0].ID == a[1].ID { + t.Error("two transitions in one commit share an id") + } + + other := build(func(c *AuthoritativeChange) { + c.Situation.InputVersion = 9 + c.OperatorArtifacts[0].AppliedInputVersion = 9 + }) + for i := range a { + if a[i].ID == other[i].ID { + t.Errorf("transition %d id did not change with the authoritative input version", i) + } + } +} + +func TestTransitionIdentityFollowsArtifactIdentity(t *testing.T) { + base := hsNext(t) + base.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + first, err := BuildTransitions(base) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + + other := hsNext(t) + other.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-2", artifactKindAnnotation, hsNow(t))} + second, err := BuildTransitions(other) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if first[0].ID == second[0].ID { + t.Error("two different artifact inputs produced the same transition id") + } +} + +func TestOperatorArtifactAuthorityIsBounded(t *testing.T) { + prior := hsFirst(t) + sum := hsPriorSummary(t, prior) + c := hsNext(t) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d transitions, want 1 (the artifact alone)", len(got)) + } + art := got[0] + + if art.Actor != model.ActorAttributedOperator { + t.Errorf("actor = %q, want attributed_operator", art.Actor) + } + if art.Lifecycle != prior.Lifecycle { + t.Errorf("an annotation changed lifecycle: %q -> %q", prior.Lifecycle, art.Lifecycle) + } + if art.Attention != prior.Attention { + t.Errorf("an annotation changed Attention: %q -> %q", prior.Attention, art.Attention) + } + if art.ActionContract.NextActor != prior.ActionContract.NextActor { + t.Errorf("an annotation changed the operator contract's next actor: %q -> %q", + prior.ActionContract.NextActor, art.ActionContract.NextActor) + } + if (art.Projection.Assessment == nil) != (prior.Projection.Assessment == nil) { + t.Fatal("an annotation changed whether an Assessment conclusion is recorded") + } + if canonicalDigest(*art.Projection.Assessment) != canonicalDigest(*prior.Projection.Assessment) { + t.Errorf("an annotation changed the Assessment conclusion: %+v -> %+v", + *prior.Projection.Assessment, *art.Projection.Assessment) + } + if art.InterruptionPriority != nil { + t.Errorf("an annotation is never a main-channel poke, got priority %q", *art.InterruptionPriority) + } + if art.Journal.AttributedActor != "operator@example.com" { + t.Errorf("attributed actor = %q, want operator@example.com", art.Journal.AttributedActor) + } + + folded, err := ProjectEpisode(&sum, art) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if len(folded.RecordedOperatorContext) != 1 { + t.Fatalf("recorded operator context = %v, want exactly one entry", folded.RecordedOperatorContext) + } + if folded.CurrentAttention != sum.CurrentAttention { + t.Errorf("an annotation changed the summary's Attention: %q -> %q", sum.CurrentAttention, folded.CurrentAttention) + } +} + +func TestOperatorArtifactAuthorityRejectsUnknownKindAndPolicyActor(t *testing.T) { + t.Run("unknown artifact kind", func(t *testing.T) { + c := hsNext(t) + bad := hsArtifact("input-1", artifactKindAnnotation, hsNow(t)) + bad.Kind = "operator_policy_recorded" + c.OperatorArtifacts = []OperatorArtifactInput{bad} + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error for an unknown artifact kind, got nil") + } + }) + + t.Run("annotation without an annotation id", func(t *testing.T) { + c := hsNext(t) + bad := hsArtifact("input-1", artifactKindAnnotation, hsNow(t)) + bad.AnnotationID = nil + c.OperatorArtifacts = []OperatorArtifactInput{bad} + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error for an annotation with no annotation id, got nil") + } + }) + + t.Run("operator_policy is never produced", func(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindVerdict, hsNow(t))} + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + for i, tr := range got { + if tr.Actor == model.ActorOperatorPolicy { + t.Errorf("transition %d used the reserved Plan 5 operator_policy actor", i) + } + if err := tr.Actor.Validate(); err != nil { + t.Errorf("transition %d actor rejected by the model: %v", i, err) + } + } + }) +} + +// ---------------------------------------------------------------------- +// Steps 4/5: the pure Episode fold. +// ---------------------------------------------------------------------- + +func TestProjectEpisodeStartsAtVersionOne(t *testing.T) { + first := hsFirst(t) + sum, err := ProjectEpisode(nil, first) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if sum.Version != 1 { + t.Errorf("version = %d, want 1", sum.Version) + } + if sum.SourceTransitionSequence != first.Sequence { + t.Errorf("source transition sequence = %d, want %d", sum.SourceTransitionSequence, first.Sequence) + } + if !sum.EffectiveStartedAt.Equal(first.Projection.EffectiveStartedAt) { + t.Errorf("effective start = %s, want the projection's %s (never CreatedAt %s)", + sum.EffectiveStartedAt, first.Projection.EffectiveStartedAt, first.CreatedAt) + } + if sum.EffectiveStartedAt.Equal(first.CreatedAt) { + t.Error("effective start must not come from the transition's creation time (R3)") + } + if sum.InitialPublicationReason != string(model.ReasonFirstAuthoritativeState) { + t.Errorf("initial publication reason = %q, want %q", sum.InitialPublicationReason, model.ReasonFirstAuthoritativeState) + } + if sum.PeakAttention != model.AttentionInvestigate || sum.CurrentAttention != model.AttentionInvestigate { + t.Errorf("attention = %q/%q, want investigate/investigate", sum.CurrentAttention, sum.PeakAttention) + } + if err := sum.Validate(); err != nil { + t.Errorf("folded summary failed model validation: %v", err) + } +} + +func TestProjectEpisodeAdvancesExactlyOncePerTransition(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{ + hsArtifact("input-1", artifactKindAnnotation, hsNow(t)), + hsArtifact("input-2", artifactKindVerdict, hsNow(t).Add(time.Second)), + } + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + + sum := *c.PriorSummary + startVersion := sum.Version + for i, tr := range got { + next, err := ProjectEpisode(&sum, tr) + if err != nil { + t.Fatalf("ProjectEpisode(%d): %v", i, err) + } + if next.Version != sum.Version+1 { + t.Fatalf("fold %d advanced version %d -> %d, want exactly one", i, sum.Version, next.Version) + } + sum = next + } + if sum.Version != startVersion+len(got) { + t.Errorf("version = %d, want %d", sum.Version, startVersion+len(got)) + } + if sum.PeakAttention != model.AttentionUrgent || sum.CurrentAttention != model.AttentionUrgent { + t.Errorf("attention = %q/%q, want urgent/urgent", sum.CurrentAttention, sum.PeakAttention) + } + if len(sum.RecordedOperatorContext) != 2 { + t.Errorf("recorded operator context = %v, want two entries", sum.RecordedOperatorContext) + } +} + +func TestProjectEpisodeKeepsPeakAttentionThroughDeEscalation(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + urgent := hsOnly(t, c) + sum, err := ProjectEpisode(c.PriorSummary, urgent) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + + back := hsNext(t) + back.Now = back.Now.Add(time.Minute) + back.Situation.InputVersion = 9 + back.PriorTransition = &urgent + back.PriorSummary = &sum + calmed := hsOnly(t, back) + if calmed.Reason != model.ReasonAttentionChanged { + t.Fatalf("reason = %q, want attention_changed", calmed.Reason) + } + final, err := ProjectEpisode(&sum, calmed) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if final.CurrentAttention != model.AttentionInvestigate { + t.Errorf("current attention = %q, want investigate", final.CurrentAttention) + } + if final.PeakAttention != model.AttentionUrgent { + t.Errorf("peak attention = %q, want urgent retained", final.PeakAttention) + } +} + +func TestProjectEpisodeTerminalDurationAndOutcome(t *testing.T) { + c := hsNext(t) + hsRecovered(&c) + c.Projection.PublicHandle = stringPtrOf("SIT-7QK2") + + tr := hsOnly(t, c) + sum, err := ProjectEpisode(c.PriorSummary, tr) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if sum.TerminalAt == nil || !sum.TerminalAt.Equal(c.Now) { + t.Fatalf("terminal at = %v, want %s", sum.TerminalAt, c.Now) + } + want := int64(c.Now.Sub(sum.EffectiveStartedAt) / time.Second) + if sum.DurationSeconds == nil || *sum.DurationSeconds != want { + t.Errorf("duration seconds = %v, want %d (terminal - effective start)", sum.DurationSeconds, want) + } + if sum.PublicHandle != "SIT-7QK2" { + t.Errorf("public handle = %q, want SIT-7QK2", sum.PublicHandle) + } + if sum.FinalOutcome == "" { + t.Error("final outcome is empty on a terminal fold") + } + if sum.RecoveryObservedAt == nil { + t.Error("recovery observation lost on the terminal fold") + } +} + +func TestProjectEpisodeClosedUnknownRecordsUncertainty(t *testing.T) { + c := hsNext(t) + hsClosedUnknown(&c) + + tr := hsOnly(t, c) + sum, err := ProjectEpisode(c.PriorSummary, tr) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if sum.RemainingUncertainty == "" { + t.Error("closed_unknown recorded no remaining uncertainty") + } + if sum.InvestigationStarted { + t.Error("a direct closed_unknown must not set InvestigationStarted") + } + if got := DeriveOrientation(sum, tr); got != OrientationClosedUncertain { + t.Errorf("orientation = %q, want %q", got, OrientationClosedUncertain) + } +} + +func TestProjectEpisodeRejectsIncoherentFolds(t *testing.T) { + first := hsFirst(t) + sum, err := ProjectEpisode(nil, first) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + + next := func() model.Transition { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + return hsOnly(t, c) + } + + t.Run("duplicate sequence", func(t *testing.T) { + tr := next() + tr.Sequence = sum.SourceTransitionSequence + if _, err := ProjectEpisode(&sum, tr); err == nil { + t.Fatal("want error for a duplicate sequence, got nil") + } + }) + + t.Run("skipped sequence", func(t *testing.T) { + tr := next() + tr.Sequence = sum.SourceTransitionSequence + 2 + if _, err := ProjectEpisode(&sum, tr); err == nil { + t.Fatal("want error for a skipped sequence, got nil") + } + }) + + t.Run("situation mismatch", func(t *testing.T) { + tr := next() + tr.SituationID = "1f0f5a0c-0000-4000-8000-00000000ffff" + if _, err := ProjectEpisode(&sum, tr); err == nil { + t.Fatal("want error for a Situation mismatch, got nil") + } + }) + + t.Run("time reversal", func(t *testing.T) { + tr := next() + tr.CreatedAt = sum.UpdatedAt.Add(-time.Second) + if _, err := ProjectEpisode(&sum, tr); err == nil { + t.Fatal("want error for a time reversal, got nil") + } + }) + + t.Run("mutation after a terminal transition", func(t *testing.T) { + c := hsNext(t) + hsRecovered(&c) + terminal := hsOnly(t, c) + terminalSummary, err := ProjectEpisode(c.PriorSummary, terminal) + if err != nil { + t.Fatalf("ProjectEpisode(terminal): %v", err) + } + + later := next() + later.Sequence = terminalSummary.SourceTransitionSequence + 1 + later.CreatedAt = terminalSummary.UpdatedAt.Add(time.Minute) + if _, err := ProjectEpisode(&terminalSummary, later); err == nil { + t.Fatal("want error for a post-terminal fold, got nil") + } + }) +} + +func TestProjectEpisodeOrientationStateMachine(t *testing.T) { + // Observed -> Investigating -> Monitoring -> Recovered, plus the refire + // return from Monitoring to Investigating. + observedChange := hsChange(t) + observedChange.Assessment.ActionContract = hsMonitoringContract(observedChange.Now.Add(time.Minute)) + observed := hsOnly(t, observedChange) + sum, err := ProjectEpisode(nil, observed) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if got := DeriveOrientation(sum, observed); got != OrientationObserved { + t.Fatalf("orientation = %q, want %q", got, OrientationObserved) + } + + step := func(prior model.Transition, priorSum model.EpisodeSummary, version int, mut func(c *AuthoritativeChange)) (model.Transition, model.EpisodeSummary) { + t.Helper() + c := hsChange(t) + c.Now = prior.CreatedAt.Add(time.Minute) + c.Situation.InputVersion = version + c.PriorTransition = &prior + c.PriorSummary = &priorSum + c.Assessment.ActionContract = hsRunningTriageContract(c.Now.Add(time.Minute)) + if mut != nil { + mut(&c) + } + tr := hsOnly(t, c) + next, err := ProjectEpisode(&priorSum, tr) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + return tr, next + } + + investigating, sum := step(observed, sum, 8, nil) + if got := DeriveOrientation(sum, investigating); got != OrientationInvestigating { + t.Fatalf("orientation = %q, want %q", got, OrientationInvestigating) + } + if !sum.InvestigationStarted { + t.Fatal("investigation start was not recorded in the summary") + } + + monitoring, sum := step(investigating, sum, 9, func(c *AuthoritativeChange) { + c.Situation.Lifecycle = model.LifecycleRecoveryPending + c.Assessment.Lifecycle = model.LifecycleRecoveryPending + c.Situation.RecoveryObservedAt = timePtr(c.Now) + c.Projection.RecoveryObservedAt = timePtr(c.Now) + }) + if got := DeriveOrientation(sum, monitoring); got != OrientationMonitoring { + t.Fatalf("orientation = %q, want %q", got, OrientationMonitoring) + } + + refired, refiredSum := step(monitoring, sum, 10, nil) + if refired.Reason != model.ReasonRecoveryFailed { + t.Fatalf("reason = %q, want recovery_failed", refired.Reason) + } + if got := DeriveOrientation(refiredSum, refired); got != OrientationInvestigating { + t.Fatalf("refire orientation = %q, want %q", got, OrientationInvestigating) + } + + recovered, recoveredSum := step(monitoring, sum, 10, hsRecovered) + if got := DeriveOrientation(recoveredSum, recovered); got != OrientationRecovered { + t.Fatalf("orientation = %q, want %q", got, OrientationRecovered) + } +} + +func TestProjectEpisodeAccumulatesInvestigationWork(t *testing.T) { + first := hsFirst(t) + sum := hsPriorSummary(t, first) + + c := hsNext(t) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + concluded := hsOnly(t, c) + sum, err := ProjectEpisode(&sum, concluded) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + if len(sum.InvestigationWork) == 0 { + t.Fatal("investigation work is empty after a conclusion") + } + if sum.EvidenceConclusion == "" { + t.Error("evidence conclusion is empty after a conclusion") + } + if sum.LatestMaterialReason != string(model.ReasonInvestigationConcluded) { + t.Errorf("latest material reason = %q, want investigation_concluded", sum.LatestMaterialReason) + } + if sum.ImpactSummary == "" { + t.Error("impact summary is empty") + } +} + +// ---------------------------------------------------------------------- +// HistoryCommit composition (the value Task 5 commits). +// ---------------------------------------------------------------------- + +func TestBuildTransitionsHistoryCommitComposition(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + + commit, err := BuildHistoryCommit(c, PublicationInput{ + Situation: c.Situation, + PriorTransition: c.PriorTransition, + RootPublished: true, + SlackFloor: model.InterruptionLow, + RepageCooldown: 15 * time.Minute, + Now: c.Now, + }) + if err != nil { + t.Fatalf("BuildHistoryCommit: %v", err) + } + if len(commit.Transitions) != 2 { + t.Fatalf("got %d transitions, want 2", len(commit.Transitions)) + } + if commit.Summary == nil { + t.Fatal("summary is nil after a material commit") + } + if want := c.PriorSummary.Version + 2; commit.Summary.Version != want { + t.Errorf("summary version = %d, want %d", commit.Summary.Version, want) + } + if len(commit.Intents) == 0 { + t.Error("no notification intents planned for a material commit") + } + + quiet, err := BuildHistoryCommit(hsNext(t), PublicationInput{ + Situation: hsNext(t).Situation, + RootPublished: true, + SlackFloor: model.InterruptionLow, + RepageCooldown: 15 * time.Minute, + Now: c.Now, + }) + if err != nil { + t.Fatalf("BuildHistoryCommit(quiet): %v", err) + } + if len(quiet.Transitions) != 0 || quiet.Summary != nil || len(quiet.Intents) != 0 { + t.Errorf("a quiet cycle produced %+v, want an empty commit", quiet) + } +} + +func TestBuildTransitionsRejectsIncoherentInput(t *testing.T) { + t.Run("prior transition from another Situation", func(t *testing.T) { + c := hsNext(t) + other := *c.PriorTransition + other.SituationID = "1f0f5a0c-0000-4000-8000-00000000ffff" + c.PriorTransition = &other + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("terminal lifecycle without a terminal instant", func(t *testing.T) { + c := hsNext(t) + c.Situation.Lifecycle = model.LifecycleRecovered + c.Assessment.Lifecycle = model.LifecycleRecovered + c.Assessment.Cadence = model.Cadence("") + c.Assessment.ActionContract = hsTerminalContract() + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("non-UTC now", func(t *testing.T) { + c := hsNext(t) + c.Now = c.Now.In(time.FixedZone("test", 3600)) + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("prior transition without a prior summary", func(t *testing.T) { + c := hsNext(t) + c.PriorSummary = nil + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error, got nil") + } + }) +} + +func TestBuildTransitionsMarksDrills(t *testing.T) { + c := hsNext(t) + c.Drill = true + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + for i, tr := range got { + if !tr.Drill { + t.Errorf("transition %d lost the Drill marker", i) + } + } +} diff --git a/internal/situation/model/history.go b/internal/situation/model/history.go index c32c206..78b59ef 100644 --- a/internal/situation/model/history.go +++ b/internal/situation/model/history.go @@ -327,10 +327,16 @@ func (p ProjectionFacts) Validate() error { return fmt.Errorf("projection_facts: %w", err) } } - switch { - case p.TerminalAt != nil && p.TerminalReason == nil: - return errors.New("projection_facts: terminal_at requires terminal_reason") - case p.TerminalAt == nil && p.TerminalReason != nil: + // A terminal instant may stand alone. Migration 0014's own lifecycle + // CHECK records a RECOVERED Situation as terminal_at NOT NULL with + // terminal_reason NULL — TerminalReason is documented as "the structured + // reason recorded when a Situation closes as closed_unknown", and its + // closed vocabulary (observation_deadline, resolution_missing, + // source_unavailable, budget_exhausted) has no value that could honestly + // describe a recovery. Requiring the pair here would make every + // `recovered` Transition unrepresentable. A reason without an instant is + // still incoherent. + if p.TerminalAt == nil && p.TerminalReason != nil { return errors.New("projection_facts: terminal_reason requires terminal_at") } if p.TerminalReason != nil { @@ -649,47 +655,9 @@ type NotificationIntent struct { DeliveredAt *time.Time } -// Validate checks NotificationIntent's required, bounded identity fields; -// its closed effect-class/status/priority codes; the effect-class-specific -// reference rules (installation_gap_recovery forbids Situation/Transition -// references and requires a gap generation; the three Situation effects -// require a Situation/Transition/summary reference according to their -// class; contract_deadline_at is accepted only on root_sync; gap_generation -// is accepted only on installation_gap_recovery); and a required, UTC -// created_at. -func (n NotificationIntent) Validate() error { - if strings.TrimSpace(n.ID) == "" { - return errors.New("notification_intent: id is required") - } - if len(n.ID) > maxIdentifierLength { - return fmt.Errorf("notification_intent: id exceeds %d bytes", maxIdentifierLength) - } - if strings.TrimSpace(n.IdempotencyKey) == "" { - return errors.New("notification_intent: idempotency_key is required") - } - if len(n.IdempotencyKey) > maxIdentifierLength { - return fmt.Errorf("notification_intent: idempotency_key exceeds %d bytes", maxIdentifierLength) - } - if err := n.EffectClass.Validate(); err != nil { - return fmt.Errorf("notification_intent: %w", err) - } - if strings.TrimSpace(n.ClientMessageID) == "" { - return errors.New("notification_intent: client_message_id is required") - } - if len(n.ClientMessageID) > maxIdentifierLength { - return fmt.Errorf("notification_intent: client_message_id exceeds %d bytes", maxIdentifierLength) - } - if err := n.Status.Validate(); err != nil { - return fmt.Errorf("notification_intent: %w", err) - } - if n.InterruptionPriority != nil { - if err := n.InterruptionPriority.Validate(); err != nil { - return fmt.Errorf("notification_intent: %w", err) - } - } - if n.AttemptCount < 0 { - return errors.New("notification_intent: attempt_count must be >= 0") - } +// validateBoundedRefs checks every nullable identifier/coordinate-shaped +// field: set means non-empty and within the identifier bound. +func (n NotificationIntent) validateBoundedRefs() error { for _, ptrField := range []struct { name string v *string @@ -710,7 +678,16 @@ func (n NotificationIntent) Validate() error { return fmt.Errorf("notification_intent: %s exceeds %d bytes", ptrField.name, maxIdentifierLength) } } + return nil +} +// validateEffectClassRefs checks the reference shape each effect class +// requires: installation_gap_recovery is installation-level (no Situation, +// Transition, or summary reference, but a gap generation), while the three +// Situation effects each reference the Situation and the Transition that +// created their content, and only root_sync additionally references the +// Episode-summary version it renders. +func (n NotificationIntent) validateEffectClassRefs() error { switch n.EffectClass { case EffectInstallationGapRecovery: if n.SituationID != nil { @@ -742,6 +719,56 @@ func (n NotificationIntent) Validate() error { return errors.New("notification_intent: root_sync requires summary_version") } } + return nil +} + +// Validate checks NotificationIntent's required, bounded identity fields; +// its closed effect-class/status/priority codes; the effect-class-specific +// reference rules (installation_gap_recovery forbids Situation/Transition +// references and requires a gap generation; the three Situation effects +// require a Situation/Transition/summary reference according to their +// class; contract_deadline_at is accepted only on root_sync; gap_generation +// is accepted only on installation_gap_recovery); and a required, UTC +// created_at. +func (n NotificationIntent) Validate() error { + if strings.TrimSpace(n.ID) == "" { + return errors.New("notification_intent: id is required") + } + if len(n.ID) > maxIdentifierLength { + return fmt.Errorf("notification_intent: id exceeds %d bytes", maxIdentifierLength) + } + if strings.TrimSpace(n.IdempotencyKey) == "" { + return errors.New("notification_intent: idempotency_key is required") + } + if len(n.IdempotencyKey) > maxIdentifierLength { + return fmt.Errorf("notification_intent: idempotency_key exceeds %d bytes", maxIdentifierLength) + } + if err := n.EffectClass.Validate(); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + if strings.TrimSpace(n.ClientMessageID) == "" { + return errors.New("notification_intent: client_message_id is required") + } + if len(n.ClientMessageID) > maxIdentifierLength { + return fmt.Errorf("notification_intent: client_message_id exceeds %d bytes", maxIdentifierLength) + } + if err := n.Status.Validate(); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + if n.InterruptionPriority != nil { + if err := n.InterruptionPriority.Validate(); err != nil { + return fmt.Errorf("notification_intent: %w", err) + } + } + if n.AttemptCount < 0 { + return errors.New("notification_intent: attempt_count must be >= 0") + } + if err := n.validateBoundedRefs(); err != nil { + return err + } + if err := n.validateEffectClassRefs(); err != nil { + return err + } if n.ContractDeadlineAt != nil { if n.EffectClass != EffectRootSync { diff --git a/internal/situation/model/history_test.go b/internal/situation/model/history_test.go index 2cf735c..c13d232 100644 --- a/internal/situation/model/history_test.go +++ b/internal/situation/model/history_test.go @@ -295,11 +295,16 @@ func TestProjectionFactsValidate(t *testing.T) { } }) - t.Run("terminal_at without terminal_reason rejected", func(t *testing.T) { + // A recovered Situation is terminal with no terminal reason: migration + // 0014's lifecycle CHECK is + // (lifecycle='recovered' AND ... terminal_at IS NOT NULL AND + // terminal_reason IS NULL), and TerminalReason's closed vocabulary only + // describes a closed_unknown closure. + t.Run("terminal_at without terminal_reason accepted (recovered)", func(t *testing.T) { p := fullProjectionFacts(now) p.TerminalAt = ptr(now) - if err := p.Validate(); err == nil { - t.Fatal("want error, got nil") + if err := p.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) } }) diff --git a/internal/situation/notification_plan.go b/internal/situation/notification_plan.go new file mode 100644 index 0000000..8254e68 --- /dev/null +++ b/internal/situation/notification_plan.go @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "errors" + "fmt" + "time" + + "github.com/google/uuid" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 4: pure notification-intent planning. Publication authority +// is decided here, inside the authoritative controller commit's own +// derivation — Slack delivery never makes a second publication decision, +// and nothing in this file renders Slack. +// ---------------------------------------------------------------------- + +// PublicationInput is one committed reconciliation plus the delivery-side +// context the publication decision needs. +type PublicationInput struct { + Situation model.Situation + // Transitions is this commit's, in sequence order; empty on a + // non-material cycle. + Transitions []model.Transition + // Summary is the Episode summary current after the fold. On a + // non-material cycle it is the unchanged current summary. + Summary model.EpisodeSummary + // PriorTransition is the Situation's current Transition before this + // commit — the authority a non-material R4 deadline refresh references, + // since such a cycle creates no Transition of its own. + PriorTransition *model.Transition + // ContractDeadlineAt is the committed nonterminal next_update_at (R4): + // the promise the root renders. Nil for a terminal commit. + ContractDeadlineAt *time.Time + RootPublished bool + LatestRootSyncVersion *int + LastDeliveredRootDeadlineAt *time.Time + LastMainChannelPokeAt *time.Time + SlackFloor model.InterruptionPriority + RepageCooldown time.Duration + Drill bool + Now time.Time +} + +// PlanNotificationIntents derives every durable Slack obligation one +// committed reconciliation creates: +// +// - one coalescible `root_sync` carrying the current Episode-summary +// version and the committed contract deadline it renders (R4); +// - one immutable `thread_append` per journaled Transition, in sequence +// order, each rendering only its own Transition's stored journal data; +// - at most one `broadcast_handoff` — the single new main-channel poke a +// commit may create, when the permitted poke class allows it, the +// repage cooldown has elapsed for the one class it gates, and the root +// is already published (an unpublished root's first post IS the poke); +// - on a non-material cycle, only the R4 deadline refresh, and only when +// the root is published, its last delivered promise has passed, and +// this commit carries a different deadline. +// +// A poke below the operator's Slack floor becomes a durable +// `withheld_by_operator_slack_floor` decision, never an absent row, and the +// floor never suppresses a non-broadcast journal entry. +func PlanNotificationIntents(in PublicationInput) ([]model.NotificationIntent, error) { + if err := validatePublicationInput(in); err != nil { + return nil, err + } + if len(in.Transitions) == 0 { + return planDeadlineRefresh(in) + } + + out := make([]model.NotificationIntent, 0, len(in.Transitions)+2) + authority := in.Transitions[len(in.Transitions)-1] + + // The root: a first publication is itself the main-channel poke; every + // later synchronization is a silent edit. + rootPoke := !in.RootPublished + root := newIntent(in, model.EffectRootSync, authority, rootSyncKey(in.Situation.ID, in.Summary.Version, authority.ID, in.ContractDeadlineAt)) + root.SummaryVersion = intPtrOf(in.Summary.Version) + if !authority.Lifecycle.Terminal() { + root.ContractDeadlineAt = in.ContractDeadlineAt + } + if rootPoke { + priority := DeriveInterruptionPriority(authority) + root.MainChannelPoke = true + root.InterruptionPriority = &priority + if !MeetsSlackFloor(priority, in.SlackFloor) { + root.Status = model.IntentWithheld + } + } + out = append(out, root) + + // Immutable journal entries, one per journaled Transition, in sequence + // order. These are never pokes and never carry a summary version. + for _, tr := range in.Transitions { + if tr.JournalKind == model.JournalNone { + continue + } + out = append(out, newIntent(in, model.EffectThreadAppend, tr, threadKey(model.EffectThreadAppend, in.Situation.ID, tr.Sequence))) + } + + // At most one new main-channel poke per commit, and none at all when + // the root post above already is one. + if !rootPoke { + if poke, ok := selectPoke(in); ok { + priority := DeriveInterruptionPriority(poke) + broadcast := newIntent(in, model.EffectBroadcastHandoff, poke, + threadKey(model.EffectBroadcastHandoff, in.Situation.ID, poke.Sequence)) + broadcast.MainChannelPoke = true + broadcast.InterruptionPriority = &priority + if !MeetsSlackFloor(priority, in.SlackFloor) { + broadcast.Status = model.IntentWithheld + } + out = append(out, broadcast) + } + } + + for i := range out { + if err := out[i].Validate(); err != nil { + return nil, fmt.Errorf("situation: planned notification intent %d: %w", i, err) + } + } + return out, nil +} + +func validatePublicationInput(in PublicationInput) error { + if in.Situation.ID == "" { + return errors.New("situation: publication input: situation id is required") + } + if in.Now.IsZero() || in.Now.Location() != time.UTC { + return fmt.Errorf("situation: publication input: now must be a non-zero UTC instant, got %s", in.Now) + } + if in.SlackFloor != "" { + if err := in.SlackFloor.Validate(); err != nil { + return fmt.Errorf("situation: publication input: %w", err) + } + } + if in.RepageCooldown < 0 { + return fmt.Errorf("situation: publication input: repage cooldown must be >= 0, got %s", in.RepageCooldown) + } + if in.Summary.SituationID != "" && in.Summary.SituationID != in.Situation.ID { + return fmt.Errorf("situation: publication input: summary belongs to situation %q, not %q", + in.Summary.SituationID, in.Situation.ID) + } + if in.LatestRootSyncVersion != nil && in.Summary.Version != 0 && in.Summary.Version < *in.LatestRootSyncVersion { + return fmt.Errorf("situation: publication input: summary version %d is older than the latest root sync version %d", + in.Summary.Version, *in.LatestRootSyncVersion) + } + if len(in.Transitions) == 0 { + return nil + } + if in.Summary.SituationID == "" || in.Summary.Version < 1 { + return errors.New("situation: publication input: a commit with transitions requires the folded episode summary") + } + for i, tr := range in.Transitions { + if tr.SituationID != in.Situation.ID { + return fmt.Errorf("situation: publication input: transition %d belongs to situation %q, not %q", + i, tr.SituationID, in.Situation.ID) + } + if i > 0 && tr.Sequence <= in.Transitions[i-1].Sequence { + return fmt.Errorf("situation: publication input: transition %d sequence %d does not follow %d", + i, tr.Sequence, in.Transitions[i-1].Sequence) + } + } + if last := in.Transitions[len(in.Transitions)-1]; in.Summary.SourceTransitionSequence != last.Sequence { + return fmt.Errorf("situation: publication input: summary source sequence %d is not this commit's last transition %d", + in.Summary.SourceTransitionSequence, last.Sequence) + } + return nil +} + +// planDeadlineRefresh implements R4's single permitted non-material effect: +// a coalescible silent root edit that replaces an expired promised-update +// time. It is never a poke and never a thread entry. +func planDeadlineRefresh(in PublicationInput) ([]model.NotificationIntent, error) { + switch { + case !in.RootPublished: + // Nothing is on screen to leave sitting on an expired promise. + return nil, nil + case in.ContractDeadlineAt == nil: + return nil, nil + case in.LastDeliveredRootDeadlineAt == nil: + return nil, nil + case in.LastDeliveredRootDeadlineAt.After(in.Now): + // The delivered promise has not passed yet. + return nil, nil + case in.ContractDeadlineAt.Equal(*in.LastDeliveredRootDeadlineAt): + return nil, nil + } + if in.PriorTransition == nil { + return nil, errors.New("situation: publication input: a published root requires its authority transition to refresh the promised update") + } + if in.Summary.Version < 1 { + return nil, errors.New("situation: publication input: a deadline refresh requires the current episode summary") + } + + refresh := newIntent(in, model.EffectRootSync, *in.PriorTransition, + rootSyncKey(in.Situation.ID, in.Summary.Version, in.PriorTransition.ID, in.ContractDeadlineAt)) + refresh.SummaryVersion = intPtrOf(in.Summary.Version) + refresh.ContractDeadlineAt = in.ContractDeadlineAt + if err := refresh.Validate(); err != nil { + return nil, fmt.Errorf("situation: planned deadline refresh: %w", err) + } + return []model.NotificationIntent{refresh}, nil +} + +// selectPoke returns the one Transition in this commit that may create a +// new main-channel poke, applying spec's closed list of permitted poke +// classes and the configured repage cooldown for the one class it gates. +// When several qualify, the highest-priority (then latest) wins: a commit +// interrupts the channel at most once. +func selectPoke(in PublicationInput) (model.Transition, bool) { + prev := in.PriorTransition + var best model.Transition + found := false + for i := range in.Transitions { + tr := in.Transitions[i] + class := ClassifyPoke(prev, tr) + prev = &in.Transitions[i] + if class == PokeNone { + continue + } + if class.CooldownApplies() && !cooldownElapsed(in) { + continue + } + if !found || !DeriveInterruptionPriority(tr).Less(DeriveInterruptionPriority(best)) { + best = tr + found = true + } + } + return best, found +} + +func cooldownElapsed(in PublicationInput) bool { + if in.LastMainChannelPokeAt == nil { + return true + } + return !in.Now.Before(in.LastMainChannelPokeAt.Add(in.RepageCooldown)) +} + +// newIntent fills the fields every Situation-scoped intent shares: +// deterministic identity, the effect's own subject references, the +// class-determined root dependency, and a pending status. +func newIntent(in PublicationInput, class model.EffectClass, subject model.Transition, key string) model.NotificationIntent { + return model.NotificationIntent{ + ID: intentIdentity("intent:" + key), + IdempotencyKey: key, + EffectClass: class, + SituationID: stringPtrOf(in.Situation.ID), + TransitionID: stringPtrOf(subject.ID), + TransitionSequence: intPtrOf(subject.Sequence), + RequiresRoot: class == model.EffectThreadAppend || class == model.EffectBroadcastHandoff, + ClientMessageID: intentIdentity("client_message:" + key), + Status: model.IntentPending, + CreatedAt: in.Now, + } +} + +// intentIdentity derives a stable UUIDv5 from the fixed AlertINT namespace +// and the intent's idempotency key, so a timeout, an uncertain success, a +// crash, and a restart all reuse the identical client message ID and +// payload identity. +func intentIdentity(scopedKey string) string { + return uuid.NewSHA1(historyNamespace, []byte(scopedKey)).String() +} + +// rootSyncKey folds the summary version and the rendered contract deadline +// into the root projection's identity (R4), so a deadline refresh is a +// distinct coalescible projection rather than a duplicate of the root it +// replaces. +func rootSyncKey(situationID string, summaryVersion int, transitionID string, deadline *time.Time) string { + stamp := "none" + if deadline != nil { + stamp = deadline.UTC().Format(time.RFC3339) + } + return boundedText(fmt.Sprintf("root_sync:%s:v%d:%s:%s", situationID, summaryVersion, transitionID, stamp), maxHistoryIdentifier) +} + +// threadKey identifies one immutable historical effect by its Transition's +// sequence, matching migration 0018's +// (situation_id, transition_sequence, effect_class) uniqueness. +func threadKey(class model.EffectClass, situationID string, sequence int) string { + return boundedText(fmt.Sprintf("%s:%s:%d", class, situationID, sequence), maxHistoryIdentifier) +} + +func intPtrOf(i int) *int { return &i } diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go new file mode 100644 index 0000000..23b8794 --- /dev/null +++ b/internal/situation/notification_plan_test.go @@ -0,0 +1,655 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// hsCommitOf runs this task's two history functions over one change: the +// derived Transitions plus the summary folded once per Transition. +func hsCommitOf(t *testing.T, c AuthoritativeChange) ([]model.Transition, model.EpisodeSummary) { + t.Helper() + trs, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + cur := model.EpisodeSummary{} + prior := c.PriorSummary + if prior != nil { + cur = *prior + } + for i, tr := range trs { + next, err := ProjectEpisode(prior, tr) + if err != nil { + t.Fatalf("ProjectEpisode(%d): %v", i, err) + } + cur = next + prior = &cur + } + return trs, cur +} + +// hsPub builds the baseline PublicationInput for a commit: an already +// published root, no floor, the plan's 15-minute repage cooldown. +func hsPub(c AuthoritativeChange, trs []model.Transition, sum model.EpisodeSummary) PublicationInput { + in := PublicationInput{ + Situation: c.Situation, + Transitions: trs, + Summary: sum, + PriorTransition: c.PriorTransition, + RootPublished: true, + SlackFloor: model.InterruptionLow, + RepageCooldown: 15 * time.Minute, + Drill: c.Drill, + Now: c.Now, + } + if len(trs) > 0 { + in.ContractDeadlineAt = trs[len(trs)-1].ActionContract.NextUpdateAt + } + return in +} + +func hsPlan(t *testing.T, in PublicationInput) []model.NotificationIntent { + t.Helper() + got, err := PlanNotificationIntents(in) + if err != nil { + t.Fatalf("PlanNotificationIntents: %v", err) + } + for i, intent := range got { + if err := intent.Validate(); err != nil { + t.Fatalf("intent %d failed model validation: %v", i, err) + } + } + return got +} + +func hsIntentsOfClass(intents []model.NotificationIntent, class model.EffectClass) []model.NotificationIntent { + out := []model.NotificationIntent{} + for _, i := range intents { + if i.EffectClass == class { + out = append(out, i) + } + } + return out +} + +// ---------------------------------------------------------------------- +// Roots. +// ---------------------------------------------------------------------- + +func TestPlanNotificationIntentsInitialRoot(t *testing.T) { + c := hsChange(t) + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + + got := hsPlan(t, in) + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1", len(roots)) + } + root := roots[0] + if !root.MainChannelPoke { + t.Error("the first warranted publication must be a main-channel poke") + } + if root.InterruptionPriority == nil { + t.Fatal("a poke intent must carry the priority it was floor-evaluated against") + } + if root.SummaryVersion == nil || *root.SummaryVersion != sum.Version { + t.Errorf("summary version = %v, want %d", root.SummaryVersion, sum.Version) + } + if root.TransitionID == nil || *root.TransitionID != trs[len(trs)-1].ID { + t.Errorf("root authority transition = %v, want %q", root.TransitionID, trs[len(trs)-1].ID) + } + if root.TransitionSequence == nil || *root.TransitionSequence != trs[len(trs)-1].Sequence { + t.Errorf("root transition sequence = %v, want %d", root.TransitionSequence, trs[len(trs)-1].Sequence) + } + if root.RequiresRoot { + t.Error("root_sync must not depend on a root") + } + if root.ContractDeadlineAt == nil || !root.ContractDeadlineAt.Equal(*in.ContractDeadlineAt) { + t.Errorf("contract deadline = %v, want %v", root.ContractDeadlineAt, in.ContractDeadlineAt) + } + if root.Status != model.IntentPending { + t.Errorf("status = %q, want pending", root.Status) + } + if root.ClientMessageID == "" { + t.Error("client message id is empty") + } + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { + t.Errorf("the first publication is the poke; got %d extra broadcasts", len(broadcasts)) + } +} + +func TestPlanNotificationIntentsLaterRootSyncIsNotAPoke(t *testing.T) { + c := hsNext(t) + hsUseReason(&c, reasonCodeDurationOutlier) + concl := *c.Projection.Assessment + concl.Impact = model.ImpactConfirmed + c.Projection.Assessment = &concl + c.Assessment.Impact = model.ImpactConfirmed + + trs, sum := hsCommitOf(t, c) + got := hsPlan(t, hsPub(c, trs, sum)) + + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1", len(roots)) + } + if roots[0].MainChannelPoke { + t.Error("a root edit is never a poke") + } + if roots[0].InterruptionPriority != nil { + t.Error("a non-poke intent must carry no interruption priority") + } +} + +func TestPlanNotificationIntentsTerminalRootCarriesNoPromise(t *testing.T) { + c := hsNext(t) + hsRecovered(&c) + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + + got := hsPlan(t, in) + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1", len(roots)) + } + if roots[0].ContractDeadlineAt != nil { + t.Errorf("a terminal root promises no update, got %v", roots[0].ContractDeadlineAt) + } + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { + t.Errorf("got %d thread entries, want 1 for the terminal transition", len(threads)) + } +} + +// ---------------------------------------------------------------------- +// R4: the deadline refresh. +// ---------------------------------------------------------------------- + +func TestPlanNotificationIntentsDeadlineRefresh(t *testing.T) { + c := hsNext(t) + base := hsPub(c, nil, model.EpisodeSummary{}) + base.Summary = *c.PriorSummary + base.ContractDeadlineAt = timePtr(c.Now.Add(5 * time.Minute)) + base.LastDeliveredRootDeadlineAt = timePtr(c.Now.Add(-time.Minute)) + base.LatestRootSyncVersion = &c.PriorSummary.Version + + t.Run("published root with an expired promise and a new deadline", func(t *testing.T) { + got := hsPlan(t, base) + if len(got) != 1 { + t.Fatalf("got %d intents, want exactly one refresh: %+v", len(got), got) + } + refresh := got[0] + if refresh.EffectClass != model.EffectRootSync { + t.Fatalf("effect class = %q, want root_sync", refresh.EffectClass) + } + if refresh.MainChannelPoke { + t.Error("a deadline refresh is never a poke") + } + if refresh.ContractDeadlineAt == nil || !refresh.ContractDeadlineAt.Equal(*base.ContractDeadlineAt) { + t.Errorf("contract deadline = %v, want %v", refresh.ContractDeadlineAt, base.ContractDeadlineAt) + } + if refresh.SummaryVersion == nil || *refresh.SummaryVersion != base.Summary.Version { + t.Errorf("summary version = %v, want the unchanged %d", refresh.SummaryVersion, base.Summary.Version) + } + if refresh.TransitionID == nil || *refresh.TransitionID != c.PriorTransition.ID { + t.Errorf("authority transition = %v, want the prior %q", refresh.TransitionID, c.PriorTransition.ID) + } + }) + + t.Run("unpublished root refreshes nothing", func(t *testing.T) { + in := base + in.RootPublished = false + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("got %d intents, want 0: %+v", len(got), got) + } + }) + + t.Run("a promise that has not passed refreshes nothing", func(t *testing.T) { + in := base + in.LastDeliveredRootDeadlineAt = timePtr(c.Now.Add(time.Minute)) + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("got %d intents, want 0: %+v", len(got), got) + } + }) + + t.Run("an unchanged deadline refreshes nothing", func(t *testing.T) { + in := base + in.ContractDeadlineAt = in.LastDeliveredRootDeadlineAt + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("got %d intents, want 0: %+v", len(got), got) + } + }) + + t.Run("a terminal commit with no deadline refreshes nothing", func(t *testing.T) { + in := base + in.ContractDeadlineAt = nil + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("got %d intents, want 0: %+v", len(got), got) + } + }) +} + +func TestPlanNotificationIntentsQuietSituationCreatesNothing(t *testing.T) { + c := hsNext(t) + in := hsPub(c, nil, *c.PriorSummary) + in.ContractDeadlineAt = timePtr(c.Now.Add(5 * time.Minute)) + in.LastDeliveredRootDeadlineAt = timePtr(c.Now.Add(4 * time.Minute)) + + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("a quiet Situation created %d Slack intents, want 0: %+v", len(got), got) + } +} + +// ---------------------------------------------------------------------- +// Immutable journal entries and handoff broadcasts. +// ---------------------------------------------------------------------- + +func TestPlanNotificationIntentsJournalAppendsInSequenceOrder(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{ + hsArtifact("input-1", artifactKindAnnotation, hsNow(t)), + hsArtifact("input-2", artifactKindVerdict, hsNow(t).Add(time.Second)), + } + trs, sum := hsCommitOf(t, c) + got := hsPlan(t, hsPub(c, trs, sum)) + + threads := hsIntentsOfClass(got, model.EffectThreadAppend) + if len(threads) != len(trs) { + t.Fatalf("got %d thread entries, want one per journaled transition (%d)", len(threads), len(trs)) + } + for i, intent := range threads { + if intent.TransitionID == nil || *intent.TransitionID != trs[i].ID { + t.Errorf("thread entry %d references %v, want %q", i, intent.TransitionID, trs[i].ID) + } + if intent.TransitionSequence == nil || *intent.TransitionSequence != trs[i].Sequence { + t.Errorf("thread entry %d sequence = %v, want %d", i, intent.TransitionSequence, trs[i].Sequence) + } + if !intent.RequiresRoot { + t.Errorf("thread entry %d must wait for durable root coordinates", i) + } + if intent.MainChannelPoke { + t.Errorf("thread entry %d must never be a poke", i) + } + if intent.SummaryVersion != nil { + t.Errorf("thread entry %d must render its own transition, not a summary version", i) + } + } +} + +func TestPlanNotificationIntentsOperatorHandoffBroadcast(t *testing.T) { + c := hsNext(t) + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + trs, sum := hsCommitOf(t, c) + got := hsPlan(t, hsPub(c, trs, sum)) + + threads := hsIntentsOfClass(got, model.EffectThreadAppend) + if len(threads) != 1 { + t.Fatalf("got %d thread entries, want exactly one journal entry", len(threads)) + } + broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff) + if len(broadcasts) != 1 { + t.Fatalf("got %d broadcast effects, want at most one (and exactly one here)", len(broadcasts)) + } + if !broadcasts[0].MainChannelPoke { + t.Error("the handoff broadcast must be the main-channel poke") + } + if broadcasts[0].InterruptionPriority == nil { + t.Fatal("the handoff broadcast carries no interruption priority") + } + if !broadcasts[0].RequiresRoot { + t.Error("a broadcast reply must wait for durable root coordinates") + } + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want the root edit that precedes the broadcast", len(roots)) + } + if roots[0].MainChannelPoke { + t.Error("the root edit for a handoff is never itself a poke") + } +} + +func TestPlanNotificationIntentsAtMostOneBroadcastPerCommit(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + trs, sum := hsCommitOf(t, c) + got := hsPlan(t, hsPub(c, trs, sum)) + + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) > 1 { + t.Fatalf("got %d broadcast effects, want at most one", len(broadcasts)) + } +} + +// ---------------------------------------------------------------------- +// Floor, cooldown, escalation. +// ---------------------------------------------------------------------- + +func TestPlanNotificationIntentsFloorWithholdsThePoke(t *testing.T) { + c := hsChange(t) + c.Situation.Attention = model.AttentionObserve + c.Assessment.Attention = model.AttentionObserve + hsUseReason(&c, reasonCodeDurationOutlier) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + in.SlackFloor = model.InterruptionHigh + + got := hsPlan(t, in) + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1 (a withheld decision, never an absent row)", len(roots)) + } + if roots[0].Status != model.IntentWithheld { + t.Errorf("status = %q, want withheld_by_operator_slack_floor", roots[0].Status) + } + if !roots[0].MainChannelPoke || roots[0].InterruptionPriority == nil { + t.Error("a withheld poke still records that it was a poke and the priority it was judged against") + } + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { + t.Errorf("the floor must never suppress a non-broadcast journal entry, got %d", len(threads)) + } + for _, intent := range hsIntentsOfClass(got, model.EffectThreadAppend) { + if intent.Status != model.IntentPending { + t.Errorf("journal entry status = %q, want pending", intent.Status) + } + } +} + +func TestPlanNotificationIntentsWithheldBroadcastKeepsTheJournal(t *testing.T) { + c := hsNext(t) + hsUseReason(&c, reasonCodeDurationOutlier) + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.SlackFloor = model.InterruptionCritical + + got := hsPlan(t, in) + broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff) + if len(broadcasts) != 1 { + t.Fatalf("got %d broadcast effects, want one withheld decision", len(broadcasts)) + } + if broadcasts[0].Status != model.IntentWithheld { + t.Errorf("status = %q, want withheld_by_operator_slack_floor", broadcasts[0].Status) + } + threads := hsIntentsOfClass(got, model.EffectThreadAppend) + if len(threads) != 1 || threads[0].Status != model.IntentPending { + t.Errorf("the journal entry must survive the floor, got %+v", threads) + } +} + +func TestPlanNotificationIntentsRepageCooldown(t *testing.T) { + handedOffChange := hsChange(t) + handedOffChange.Assessment.ActionContract = hsOperatorContract(handedOffChange.Now.Add(time.Minute)) + handedOff := hsOnly(t, handedOffChange) + sum, err := ProjectEpisode(nil, handedOff) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + + changeAt := func(offset time.Duration, mut func(c *AuthoritativeChange)) AuthoritativeChange { + c := hsChange(t) + c.Now = handedOff.CreatedAt.Add(offset) + c.Situation.InputVersion = 9 + c.PriorTransition = &handedOff + c.PriorSummary = &sum + contract := hsOperatorContract(c.Now.Add(time.Minute)) + contract.NextUpdateOn = []model.NextUpdateOn{model.NextUpdateOnSourceResolution} + c.Assessment.ActionContract = contract + if mut != nil { + mut(&c) + } + return c + } + + t.Run("a changed required action inside the cooldown does not repage", func(t *testing.T) { + c := changeAt(2*time.Minute, nil) + trs, folded := hsCommitOf(t, c) + in := hsPub(c, trs, folded) + in.LastMainChannelPokeAt = timePtr(handedOff.CreatedAt) + + got := hsPlan(t, in) + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { + t.Errorf("got %d broadcasts inside the cooldown, want 0", len(broadcasts)) + } + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { + t.Errorf("the journal entry is never cooled down, got %d", len(threads)) + } + }) + + t.Run("a changed required action after the cooldown repages", func(t *testing.T) { + c := changeAt(20*time.Minute, nil) + trs, folded := hsCommitOf(t, c) + in := hsPub(c, trs, folded) + in.LastMainChannelPokeAt = timePtr(handedOff.CreatedAt) + + got := hsPlan(t, in) + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 1 { + t.Errorf("got %d broadcasts after the cooldown, want 1", len(broadcasts)) + } + }) + + t.Run("escalation bypasses the cooldown", func(t *testing.T) { + c := changeAt(2*time.Minute, func(c *AuthoritativeChange) { + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + }) + trs, folded := hsCommitOf(t, c) + in := hsPub(c, trs, folded) + in.LastMainChannelPokeAt = timePtr(handedOff.CreatedAt) + + got := hsPlan(t, in) + broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff) + if len(broadcasts) != 1 { + t.Fatalf("got %d broadcasts, want 1 escalation that bypasses the cooldown", len(broadcasts)) + } + if broadcasts[0].InterruptionPriority == nil || *broadcasts[0].InterruptionPriority != model.InterruptionCritical { + t.Errorf("escalation priority = %v, want critical", broadcasts[0].InterruptionPriority) + } + }) +} + +func TestPlanNotificationIntentsCriticalPublicationWithoutL2(t *testing.T) { + // The L2 provider is unavailable, so the authoritative Assessment is a + // deterministic fallback — the critical floor still publishes and still + // pokes, at critical priority, past any configured floor. + c := hsChange(t) + c.Derivation = model.DerivationDeterministicFallback + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + concl := *c.Projection.Assessment + concl.EvidenceQuality = model.EvidenceQualityDegraded + concl.LimitationCodes = []string{"semantic_assessment_unavailable"} + c.Projection.Assessment = &concl + c.Assessment.EvidenceQuality = model.EvidenceQualityDegraded + + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + in.SlackFloor = model.InterruptionHigh + + got := hsPlan(t, in) + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1", len(roots)) + } + if roots[0].Status != model.IntentPending { + t.Errorf("status = %q, want pending — critical always passes the floor", roots[0].Status) + } + if roots[0].InterruptionPriority == nil || *roots[0].InterruptionPriority != model.InterruptionCritical { + t.Errorf("priority = %v, want critical", roots[0].InterruptionPriority) + } + if trs[0].Actor != model.ActorDeterministicController { + t.Errorf("actor = %q, want deterministic_controller for a fallback Assessment", trs[0].Actor) + } +} + +// ---------------------------------------------------------------------- +// Recurrence, recovery, refire, drills, identity. +// ---------------------------------------------------------------------- + +func TestPlanNotificationIntentsRecurrenceMilestoneStaysInThread(t *testing.T) { + c := hsNext(t) + c.RecurrenceCount = 10 + trs, sum := hsCommitOf(t, c) + got := hsPlan(t, hsPub(c, trs, sum)) + + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { + t.Errorf("a recurrence milestone must never broadcast, got %d", len(broadcasts)) + } + threads := hsIntentsOfClass(got, model.EffectThreadAppend) + if len(threads) != 1 { + t.Fatalf("got %d thread entries, want 1", len(threads)) + } + if threads[0].MainChannelPoke { + t.Error("a recurrence milestone is never a poke") + } + if sum.RecurrenceCount != 10 { + t.Errorf("summary recurrence count = %d, want 10", sum.RecurrenceCount) + } +} + +func TestPlanNotificationIntentsRecoveryAndRefire(t *testing.T) { + recoveringChange := hsChange(t) + recoveringChange.Situation.Lifecycle = model.LifecycleRecoveryPending + recoveringChange.Assessment.Lifecycle = model.LifecycleRecoveryPending + recoveringChange.Situation.RecoveryObservedAt = timePtr(recoveringChange.Now) + recoveringChange.Projection.RecoveryObservedAt = timePtr(recoveringChange.Now) + monitoring := hsOnly(t, recoveringChange) + sum, err := ProjectEpisode(nil, monitoring) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + + refire := hsChange(t) + refire.Now = monitoring.CreatedAt.Add(time.Minute) + refire.Situation.InputVersion = 8 + refire.PriorTransition = &monitoring + refire.PriorSummary = &sum + trs, folded := hsCommitOf(t, refire) + if trs[0].Reason != model.ReasonRecoveryFailed { + t.Fatalf("reason = %q, want recovery_failed", trs[0].Reason) + } + + got := hsPlan(t, hsPub(refire, trs, folded)) + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { + t.Fatalf("got %d thread entries, want 1 refire journal entry", len(threads)) + } + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { + t.Errorf("a refire is not on the permitted poke list, got %d broadcasts", len(broadcasts)) + } + if roots := hsIntentsOfClass(got, model.EffectRootSync); len(roots) != 1 { + t.Errorf("got %d root_sync intents, want 1", len(roots)) + } +} + +func TestPlanNotificationIntentsDrillCommitPlansNormally(t *testing.T) { + c := hsNext(t) + c.Drill = true + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + trs, sum := hsCommitOf(t, c) + if !trs[0].Drill { + t.Fatal("the derived transition lost the Drill marker") + } + got := hsPlan(t, hsPub(c, trs, sum)) + if len(got) == 0 { + t.Fatal("a drill commit planned no intents") + } + for _, intent := range got { + if intent.TransitionID == nil { + continue + } + if *intent.TransitionID != trs[0].ID && *intent.TransitionID != trs[len(trs)-1].ID { + t.Errorf("intent references %q, outside this drill commit", *intent.TransitionID) + } + } +} + +func TestPlanNotificationIntentsIdentityIsDeterministic(t *testing.T) { + plan := func() []model.NotificationIntent { + c := hsNext(t) + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + trs, sum := hsCommitOf(t, c) + return hsPlan(t, hsPub(c, trs, sum)) + } + a, b := plan(), plan() + if len(a) != len(b) || len(a) == 0 { + t.Fatalf("plans differ in size: %d vs %d", len(a), len(b)) + } + seen := map[string]bool{} + for i := range a { + if a[i].ID != b[i].ID { + t.Errorf("intent %d id is not deterministic: %q vs %q", i, a[i].ID, b[i].ID) + } + if a[i].IdempotencyKey != b[i].IdempotencyKey { + t.Errorf("intent %d idempotency key is not deterministic: %q vs %q", i, a[i].IdempotencyKey, b[i].IdempotencyKey) + } + if a[i].ClientMessageID != b[i].ClientMessageID { + t.Errorf("intent %d client message id is not deterministic: %q vs %q", i, a[i].ClientMessageID, b[i].ClientMessageID) + } + if seen[a[i].IdempotencyKey] { + t.Errorf("duplicate idempotency key %q in one commit", a[i].IdempotencyKey) + } + seen[a[i].IdempotencyKey] = true + } + + // A refreshed deadline is a distinct root projection, not the same one. + c := hsNext(t) + in := hsPub(c, nil, *c.PriorSummary) + in.ContractDeadlineAt = timePtr(c.Now.Add(5 * time.Minute)) + in.LastDeliveredRootDeadlineAt = timePtr(c.Now.Add(-time.Minute)) + first := hsPlan(t, in) + in.ContractDeadlineAt = timePtr(c.Now.Add(9 * time.Minute)) + second := hsPlan(t, in) + if len(first) != 1 || len(second) != 1 { + t.Fatalf("want one refresh each, got %d and %d", len(first), len(second)) + } + if first[0].IdempotencyKey == second[0].IdempotencyKey { + t.Error("two different deadlines produced the same root_sync idempotency key") + } +} + +func TestPlanNotificationIntentsRejectsIncoherentInput(t *testing.T) { + c := hsNext(t) + trs, sum := hsCommitOf(t, c) + _ = trs + + t.Run("summary from another Situation", func(t *testing.T) { + in := hsPub(c, nil, sum) + in.Summary.SituationID = "1f0f5a0c-0000-4000-8000-00000000ffff" + if _, err := PlanNotificationIntents(in); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("unknown Slack floor", func(t *testing.T) { + in := hsPub(c, nil, *c.PriorSummary) + in.SlackFloor = model.InterruptionPriority("shout") + if _, err := PlanNotificationIntents(in); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("transitions without a prior authority and no summary", func(t *testing.T) { + urgent := hsNext(t) + urgent.Situation.Attention = model.AttentionUrgent + urgent.Assessment.Attention = model.AttentionUrgent + built, _ := hsCommitOf(t, urgent) + in := hsPub(urgent, built, model.EpisodeSummary{}) + if _, err := PlanNotificationIntents(in); err == nil { + t.Fatal("want error for transitions with no folded summary, got nil") + } + }) +} diff --git a/internal/situation/priority.go b/internal/situation/priority.go new file mode 100644 index 0000000..477c4aa --- /dev/null +++ b/internal/situation/priority.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import "github.com/alertint/alertint-agent/internal/situation/model" + +// ---------------------------------------------------------------------- +// Plan 3 Task 4: deterministic Interruption priority and main-channel poke +// classification (spec.md "Publication authority and Interruption +// priority"). Both are pure functions of one already-derived Transition +// (plus, for the poke class, the Transition it followed): the controller +// decides publication authority, and Slack delivery never makes a second +// publication decision. +// ---------------------------------------------------------------------- + +// DeriveInterruptionPriority ranks one Transition on the closed +// deterministic scale the operator's `notify.slack.min_severity` floor is +// compared against. It is never Alert severity and never model-authored +// severity: +// +// critical an unquieted deterministic critical floor +// high urgent Attention, operator judgment/action required, or +// actionable terminal uncertainty +// medium non-critical warranted investigation while AlertINT remains +// the next actor +// low informational standalone interruption +// +// "Unquieted" is the freshness test: a deterministic critical floor only +// ranks critical while the Situation is still nonterminal — a recovered +// Situation's historical criticality never re-pages. Recovery/refire and +// recurrence reach the poke decision through ClassifyPoke (a refire is +// ranked by the Attention and contract it lands in; a recurrence milestone +// is never a poke at all), not by inflating this rank. +func DeriveInterruptionPriority(t model.Transition) model.InterruptionPriority { + terminal := t.Lifecycle.Terminal() + switch { + case !terminal && deterministicCriticalFloor(t): + return model.InterruptionCritical + case !terminal && t.Attention == model.AttentionUrgent: + return model.InterruptionHigh + case !terminal && t.ActionContract.OperatorActionRequired != nil: + return model.InterruptionHigh + case t.Lifecycle == model.LifecycleClosedUnknown: + // Actionable terminal uncertainty: AlertINT closed the Situation + // without being able to confirm resolution. + return model.InterruptionHigh + case !terminal && t.Attention == model.AttentionInvestigate && t.ActionContract.NextActor == model.NextActorAlertINT: + return model.InterruptionMedium + default: + return model.InterruptionLow + } +} + +// deterministicCriticalFloor reports whether the Transition's accepted +// Sufficient reason is the catalog's deterministic critical floor +// (reasons.go's critical_anchor — the only DeterministicFloor candidate +// Plan 2 can reach). +func deterministicCriticalFloor(t model.Transition) bool { + return t.Projection.Assessment != nil && t.Projection.Assessment.SufficientReasonCode == reasonCodeCriticalAnchor +} + +// MeetsSlackFloor reports whether priority is at or above the operator's +// configured minimum Interruption priority. An empty floor is "no floor". +// critical always passes. +func MeetsSlackFloor(priority, floor model.InterruptionPriority) bool { + return !priority.Less(floor) +} + +// PokeClass names which entry on spec.md's closed list of permitted new +// main-channel pokes a Transition qualifies for, or PokeNone. +type PokeClass string + +const ( + // PokeNone means this Transition may never create a new main-channel + // poke. Root edits and non-broadcast replies are never pokes. + PokeNone PokeClass = "" + // PokeFirstPublication is the Situation's first warranted publication. + PokeFirstPublication PokeClass = "first_publication" + // PokeCriticalityCrossed is newly crossed deterministic criticality. + PokeCriticalityCrossed PokeClass = "criticality_crossed" + // PokeUrgentAttention is newly valid urgent Attention. + PokeUrgentAttention PokeClass = "urgent_attention" + // PokeOperatorHandoff is a no-action to operator-judgment/action handoff. + PokeOperatorHandoff PokeClass = "operator_handoff" + // PokeRequiredActionChanged is a materially changed required action; it + // is the one class the configured repage cooldown gates. + PokeRequiredActionChanged PokeClass = "required_action_changed" +) + +// CooldownApplies reports whether this poke class must wait out the +// configured repage cooldown after the last delivered main-channel poke. +// Only a materially changed required action does; every escalation class +// (first publication, newly crossed criticality, newly urgent Attention, a +// handoff) bypasses it, because a cooldown must never swallow an +// escalation. +func (c PokeClass) CooldownApplies() bool { return c == PokeRequiredActionChanged } + +// ClassifyPoke reports which permitted poke class t qualifies for, given +// the Transition it directly follows (nil when t is the Situation's first +// Transition). It answers the class question only: whether the poke is +// actually emitted also depends on the repage cooldown, the operator's +// Slack floor, and whether the root is already published — all of which +// PlanNotificationIntents owns. +func ClassifyPoke(prior *model.Transition, t model.Transition) PokeClass { + switch t.Reason { //nolint:exhaustive // only these two reasons are categorically ineligible; every other reason continues to the class tests below. + case model.ReasonOperatorArtifactRecorded: + // An attributed annotation or Captured verdict grants no + // publication authority (spec.md "Domain model"). + return PokeNone + case model.ReasonRecurrenceMilestone: + // Recurrence stays in the owning Situation thread; a flapper never + // re-pages the channel (ADR-0020). + return PokeNone + } + if prior == nil { + return PokeFirstPublication + } + if t.Lifecycle.Terminal() { + // Recovery and closure are reported by editing the root and + // appending the journal, never by a new interruption. + return PokeNone + } + switch { + case deterministicCriticalFloor(t) && !deterministicCriticalFloor(*prior): + return PokeCriticalityCrossed + case t.Attention == model.AttentionUrgent && prior.Attention != model.AttentionUrgent: + return PokeUrgentAttention + case t.ActionContract.OperatorActionRequired != nil && prior.ActionContract.OperatorActionRequired == nil: + return PokeOperatorHandoff + case t.ActionContract.OperatorActionRequired != nil && + operatorContractTuple(prior.ActionContract) != operatorContractTuple(t.ActionContract): + return PokeRequiredActionChanged + default: + return PokeNone + } +} diff --git a/internal/situation/priority_test.go b/internal/situation/priority_test.go new file mode 100644 index 0000000..1e7d46f --- /dev/null +++ b/internal/situation/priority_test.go @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// hsTransitionFor builds one controller-state Transition from a mutated +// baseline change — the unit both the priority rank table and the poke +// classifier are exercised against. +func hsTransitionFor(t *testing.T, mut func(c *AuthoritativeChange)) model.Transition { + t.Helper() + c := hsNext(t) + if mut != nil { + mut(&c) + } + return hsOnly(t, c) +} + +func TestInterruptionPriorityRanks(t *testing.T) { + cases := []struct { + name string + mut func(c *AuthoritativeChange) + want model.InterruptionPriority + }{ + { + name: "unquieted deterministic critical floor", + mut: func(c *AuthoritativeChange) { + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + }, + want: model.InterruptionCritical, + }, + { + name: "urgent attention without a critical floor", + mut: func(c *AuthoritativeChange) { + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + hsUseReason(c, reasonCodeDurationOutlier) + }, + want: model.InterruptionHigh, + }, + { + name: "operator action required", + mut: func(c *AuthoritativeChange) { + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + hsUseReason(c, reasonCodeDurationOutlier) + }, + want: model.InterruptionHigh, + }, + { + name: "actionable terminal uncertainty", + mut: func(c *AuthoritativeChange) { + hsClosedUnknown(c) + }, + want: model.InterruptionHigh, + }, + { + name: "warranted investigation while AlertINT acts next", + mut: func(c *AuthoritativeChange) { + hsUseReason(c, reasonCodeDurationOutlier) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + }, + want: model.InterruptionMedium, + }, + { + name: "informational observe state", + mut: func(c *AuthoritativeChange) { + c.Situation.Attention = model.AttentionObserve + c.Assessment.Attention = model.AttentionObserve + hsUseReason(c, reasonCodeDurationOutlier) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + }, + want: model.InterruptionLow, + }, + { + name: "a quieted critical floor after recovery", + mut: hsRecovered, + want: model.InterruptionLow, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := DeriveInterruptionPriority(hsTransitionFor(t, tc.mut)) + if got != tc.want { + t.Errorf("priority = %q, want %q", got, tc.want) + } + if err := got.Validate(); err != nil { + t.Errorf("derived priority rejected by the model: %v", err) + } + }) + } +} + +func TestInterruptionPriorityPokeClasses(t *testing.T) { + // Baseline prior: a published, non-critical Situation — duration_outlier + // Sufficient reason, investigate Attention, AlertINT running Acute + // Triage — so "newly crossed criticality" is a real crossing. + priorChange := hsChange(t) + hsUseReason(&priorChange, reasonCodeDurationOutlier) + prior := hsOnly(t, priorChange) + priorSum := hsPriorSummary(t, prior) + + next := func(t *testing.T, mut func(c *AuthoritativeChange)) model.Transition { + t.Helper() + c := hsChange(t) + hsUseReason(&c, reasonCodeDurationOutlier) + c.Now = prior.CreatedAt.Add(time.Minute) + c.Situation.InputVersion = 8 + c.PriorTransition = &prior + c.PriorSummary = &priorSum + c.Assessment.ActionContract = hsRunningTriageContract(c.Now.Add(time.Minute)) + if mut != nil { + mut(&c) + } + return hsOnly(t, c) + } + + cases := []struct { + name string + firstEver bool + mut func(c *AuthoritativeChange) + want PokeClass + wantCooled bool + }{ + { + name: "first warranted publication", + firstEver: true, + want: PokeFirstPublication, + }, + { + name: "newly crossed deterministic criticality", + mut: func(c *AuthoritativeChange) { + hsUseReason(c, reasonCodeCriticalAnchor) + }, + want: PokeCriticalityCrossed, + }, + { + name: "valid urgent Attention", + mut: func(c *AuthoritativeChange) { + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + }, + want: PokeUrgentAttention, + }, + { + name: "no-action to operator handoff", + mut: func(c *AuthoritativeChange) { + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + }, + want: PokeOperatorHandoff, + }, + { + name: "ordinary investigation progress is never a poke", + mut: func(c *AuthoritativeChange) { + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + }, + want: PokeNone, + }, + { + name: "recurrence milestones never poke", + mut: func(c *AuthoritativeChange) { + c.RecurrenceCount = 25 + }, + want: PokeNone, + }, + { + name: "recovery never pokes", + mut: hsRecovered, + want: PokeNone, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var tr model.Transition + var priorArg *model.Transition + if tc.firstEver { + c := hsChange(t) + hsUseReason(&c, reasonCodeDurationOutlier) + if tc.mut != nil { + tc.mut(&c) + } + tr = hsOnly(t, c) + } else { + priorArg = &prior + tr = next(t, tc.mut) + } + if got := ClassifyPoke(priorArg, tr); got != tc.want { + t.Errorf("poke class = %q, want %q", got, tc.want) + } + }) + } +} + +func TestInterruptionPriorityRequiredActionChangeIsCooldownGated(t *testing.T) { + handedOffChange := hsChange(t) + handedOffChange.Assessment.ActionContract = hsOperatorContract(handedOffChange.Now.Add(time.Minute)) + handedOff := hsOnly(t, handedOffChange) + sum, err := ProjectEpisode(nil, handedOff) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + + changed := hsChange(t) + changed.Now = handedOff.CreatedAt.Add(time.Minute) + changed.Situation.InputVersion = 9 + changed.PriorTransition = &handedOff + changed.PriorSummary = &sum + contract := hsOperatorContract(changed.Now.Add(time.Minute)) + contract.NextUpdateOn = []model.NextUpdateOn{model.NextUpdateOnSourceResolution} + changed.Assessment.ActionContract = contract + next := hsOnly(t, changed) + + if got := ClassifyPoke(&handedOff, next); got != PokeRequiredActionChanged { + t.Fatalf("poke class = %q, want %q", got, PokeRequiredActionChanged) + } + if !PokeRequiredActionChanged.CooldownApplies() { + t.Error("a materially changed required action must respect the repage cooldown") + } + for _, escalation := range []PokeClass{PokeFirstPublication, PokeCriticalityCrossed, PokeUrgentAttention, PokeOperatorHandoff} { + if escalation.CooldownApplies() { + t.Errorf("%q must bypass the repage cooldown", escalation) + } + } +} + +func TestInterruptionPriorityFloorComparison(t *testing.T) { + cases := []struct { + priority, floor model.InterruptionPriority + want bool + }{ + {model.InterruptionCritical, model.InterruptionHigh, true}, + {model.InterruptionCritical, model.InterruptionCritical, true}, + {model.InterruptionHigh, model.InterruptionHigh, true}, + {model.InterruptionMedium, model.InterruptionHigh, false}, + {model.InterruptionLow, model.InterruptionMedium, false}, + {model.InterruptionLow, model.InterruptionLow, true}, + {model.InterruptionLow, model.InterruptionPriority(""), true}, + } + for _, tc := range cases { + if got := MeetsSlackFloor(tc.priority, tc.floor); got != tc.want { + t.Errorf("MeetsSlackFloor(%q, %q) = %v, want %v", tc.priority, tc.floor, got, tc.want) + } + } +} + +func TestInterruptionPriorityArtifactsNeverPoke(t *testing.T) { + c := hsNext(t) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d transitions, want 1", len(got)) + } + if class := ClassifyPoke(c.PriorTransition, got[0]); class != PokeNone { + t.Errorf("artifact poke class = %q, want %q", class, PokeNone) + } +} From de91a2e0d3752841a4bc5a246e03d2f96fca12ac Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 01:56:53 +0300 Subject: [PATCH 06/31] fix(situation): keep the escalation poke when artifacts journal first Two review findings on 86b8ea4: - selectPoke advanced its comparison basis through every Transition in the commit, including operator_artifact_recorded ones. An artifact Transition copies the commit's NEW lifecycle/Attention/contract verbatim, so the controller-state Transition (always last, per R1) was compared against itself and every escalation collapsed to PokeNone: an operator annotating a Situation in the cycle it escalated to urgent silently lost the poke, while the durable Transition still recorded a poke-eligible priority. Every Transition is now classified against the pre-commit state. - Relaxing ProjectionFacts.Validate to accept a bare terminal_at (correct for `recovered`) left nothing enforcing closed_unknown => terminal reason. A closed_unknown Transition with no reason folded into the Episode summary as "Recovered without recorded operator intervention". validateChange now mirrors migration 0014's lifecycle CHECK in both directions. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/situation/history.go | 22 +++++ internal/situation/history_test.go | 26 ++++++ internal/situation/notification_plan.go | 14 ++- internal/situation/notification_plan_test.go | 89 ++++++++++++++++++-- internal/situation/priority_test.go | 40 +++++++++ 5 files changed, 179 insertions(+), 12 deletions(-) diff --git a/internal/situation/history.go b/internal/situation/history.go index cd43f99..321a3ed 100644 --- a/internal/situation/history.go +++ b/internal/situation/history.go @@ -211,6 +211,28 @@ func validateChange(change AuthoritativeChange) error { return fmt.Errorf("situation: authoritative change: lifecycle %q and projection terminal_at %v disagree", change.Situation.Lifecycle, change.Projection.TerminalAt) } + // Which terminal lifecycle carries a terminal reason, mirroring + // migration 0014's own lifecycle CHECK in both directions. + // ProjectionFacts.Validate deliberately allows a bare terminal_at (a + // `recovered` Situation has no terminal reason), so this is the only + // place that catches a closed_unknown with no reason — which would + // otherwise fold into the Episode summary as a clean recovery, since + // the outcome/uncertainty text keys off the terminal reason. + switch change.Situation.Lifecycle { + case model.LifecycleClosedUnknown: + if change.Projection.TerminalReason == nil { + return errors.New("situation: authoritative change: closed_unknown requires projection terminal_reason") + } + case model.LifecycleRecovered: + if change.Projection.TerminalReason != nil { + return fmt.Errorf("situation: authoritative change: recovered must not set projection terminal_reason, got %q", + *change.Projection.TerminalReason) + } + case model.LifecycleActive, model.LifecycleRecoveryPending: + // Nonterminal: the terminal_at agreement check above already + // guarantees no terminal instant, and ProjectionFacts.Validate + // guarantees no reason without an instant. + } if err := change.Projection.Validate(); err != nil { return fmt.Errorf("situation: authoritative change: %w", err) } diff --git a/internal/situation/history_test.go b/internal/situation/history_test.go index 7083a8f..7ca9a97 100644 --- a/internal/situation/history_test.go +++ b/internal/situation/history_test.go @@ -1391,6 +1391,32 @@ func TestBuildTransitionsRejectsIncoherentInput(t *testing.T) { } }) + // Migration 0014's lifecycle CHECK, both directions: closed_unknown + // carries a terminal reason, recovered never does. ProjectionFacts + // alone cannot enforce this (it has no lifecycle), and without it a + // closed_unknown with no reason folds into the Episode summary as a + // clean recovery. + t.Run("closed_unknown without a terminal reason", func(t *testing.T) { + c := hsNext(t) + hsClosedUnknown(&c) + c.Situation.TerminalReason = nil + c.Projection.TerminalReason = nil + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error, got nil") + } + }) + + t.Run("recovered with a terminal reason", func(t *testing.T) { + c := hsNext(t) + hsRecovered(&c) + reason := model.TerminalReasonObservationDeadline + c.Situation.TerminalReason = &reason + c.Projection.TerminalReason = &reason + if _, err := BuildTransitions(c); err == nil { + t.Fatal("want error, got nil") + } + }) + t.Run("non-UTC now", func(t *testing.T) { c := hsNext(t) c.Now = c.Now.In(time.FixedZone("test", 3600)) diff --git a/internal/situation/notification_plan.go b/internal/situation/notification_plan.go index 8254e68..5cbb332 100644 --- a/internal/situation/notification_plan.go +++ b/internal/situation/notification_plan.go @@ -213,13 +213,21 @@ func planDeadlineRefresh(in PublicationInput) ([]model.NotificationIntent, error // When several qualify, the highest-priority (then latest) wins: a commit // interrupts the channel at most once. func selectPoke(in PublicationInput) (model.Transition, bool) { - prev := in.PriorTransition var best model.Transition found := false for i := range in.Transitions { tr := in.Transitions[i] - class := ClassifyPoke(prev, tr) - prev = &in.Transitions[i] + // Every Transition is classified against the state BEFORE this + // commit, never against an earlier Transition of the same commit. + // An `operator_artifact_recorded` Transition copies this commit's + // new lifecycle/Attention/contract verbatim (it changes none of + // them), so advancing the comparison basis through one would make + // the controller-state Transition — always last, per R1 — compare + // new state against itself and silently swallow the escalation it + // is entitled to. This also keeps the plan's poke decision + // identical to the InterruptionPriority already stamped on the + // durable Transition by controllerTransition. + class := ClassifyPoke(in.PriorTransition, tr) if class == PokeNone { continue } diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go index 23b8794..d387fd1 100644 --- a/internal/situation/notification_plan_test.go +++ b/internal/situation/notification_plan_test.go @@ -316,17 +316,88 @@ func TestPlanNotificationIntentsOperatorHandoffBroadcast(t *testing.T) { } } +// TestPlanNotificationIntentsAtMostOneBroadcastPerCommit pins BOTH bounds: +// several qualifying escalations in one commit still interrupt the channel +// only once, and a commit that qualifies at all always produces that one +// broadcast — including when operator artifacts are journaled ahead of the +// controller-state Transition in the same commit (R1). An artifact +// Transition copies the commit's new state verbatim, so classifying the +// controller-state Transition against it would compare new state with +// itself and silently swallow the poke. func TestPlanNotificationIntentsAtMostOneBroadcastPerCommit(t *testing.T) { - c := hsNext(t) - c.Situation.Attention = model.AttentionUrgent - c.Assessment.Attention = model.AttentionUrgent - c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) - c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} - trs, sum := hsCommitOf(t, c) - got := hsPlan(t, hsPub(c, trs, sum)) + build := func(t *testing.T, artifacts ...OperatorArtifactInput) []model.NotificationIntent { + t.Helper() + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + c.OperatorArtifacts = artifacts + trs, sum := hsCommitOf(t, c) + return hsPlan(t, hsPub(c, trs, sum)) + } + + cases := []struct { + name string + artifacts []OperatorArtifactInput + }{ + {name: "no pending artifacts"}, + {name: "one pending artifact", artifacts: []OperatorArtifactInput{ + hsArtifact("input-1", artifactKindAnnotation, hsNow(t)), + }}, + {name: "two pending artifacts", artifacts: []OperatorArtifactInput{ + hsArtifact("input-1", artifactKindAnnotation, hsNow(t)), + hsArtifact("input-2", artifactKindVerdict, hsNow(t).Add(time.Second)), + }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := build(t, tc.artifacts...) + broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff) + if len(broadcasts) != 1 { + t.Fatalf("got %d broadcast effects, want exactly one escalation poke", len(broadcasts)) + } + if !broadcasts[0].MainChannelPoke || broadcasts[0].InterruptionPriority == nil { + t.Error("the escalation broadcast must be a poke carrying its evaluated priority") + } + // The broadcast must name the controller-state Transition (last + // in the commit, per R1) — the same authority the root_sync + // references — never an artifact Transition. + roots := hsIntentsOfClass(got, model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1", len(roots)) + } + if broadcasts[0].TransitionID == nil || roots[0].TransitionID == nil || + *broadcasts[0].TransitionID != *roots[0].TransitionID { + t.Errorf("broadcast references %v, want the commit's authority transition %v", + broadcasts[0].TransitionID, roots[0].TransitionID) + } + }) + } +} - if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) > 1 { - t.Fatalf("got %d broadcast effects, want at most one", len(broadcasts)) +// TestPlanNotificationIntentsEscalationSurvivesJournaledArtifacts is the +// direct regression for the same defect, stated as a comparison: the same +// escalation must produce the same poke with and without an operator +// artifact journaled ahead of it in the same commit. +func TestPlanNotificationIntentsEscalationSurvivesJournaledArtifacts(t *testing.T) { + escalate := func(t *testing.T, artifacts []OperatorArtifactInput) int { + t.Helper() + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = artifacts + trs, sum := hsCommitOf(t, c) + return len(hsIntentsOfClass(hsPlan(t, hsPub(c, trs, sum)), model.EffectBroadcastHandoff)) + } + + without := escalate(t, nil) + with := escalate(t, []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))}) + if without != 1 { + t.Fatalf("urgent escalation alone produced %d broadcasts, want 1", without) + } + if with != without { + t.Errorf("a journaled operator artifact changed the escalation poke: %d broadcasts with, %d without", with, without) } } diff --git a/internal/situation/priority_test.go b/internal/situation/priority_test.go index 1e7d46f..a130e98 100644 --- a/internal/situation/priority_test.go +++ b/internal/situation/priority_test.go @@ -250,6 +250,46 @@ func TestInterruptionPriorityFloorComparison(t *testing.T) { } } +// TestInterruptionPriorityArtifactSiblingIsNotAComparisonBasis pins why +// selectPoke classifies every Transition against the state BEFORE the +// commit rather than against the preceding Transition of the same commit. +// An `operator_artifact_recorded` Transition copies the commit's NEW +// lifecycle/Attention/contract verbatim (it changes none of them), so using +// it as the comparison basis makes the controller-state Transition compare +// new state against itself — no crossing, no escalation, no poke. +func TestInterruptionPriorityArtifactSiblingIsNotAComparisonBasis(t *testing.T) { + c := hsNext(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + got, err := BuildTransitions(c) + if err != nil { + t.Fatalf("BuildTransitions: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d transitions, want the artifact plus the escalation", len(got)) + } + artifact, escalation := got[0], got[1] + + if artifact.Attention != escalation.Attention { + t.Fatalf("fixture no longer models the hazard: artifact attention %q vs escalation %q", + artifact.Attention, escalation.Attention) + } + if class := ClassifyPoke(&artifact, escalation); class != PokeNone { + t.Errorf("classifying against a same-commit artifact sibling = %q; the characterization this "+ + "test guards has changed, so recheck selectPoke", class) + } + if class := ClassifyPoke(c.PriorTransition, escalation); class != PokeUrgentAttention { + t.Errorf("classifying against the true pre-commit prior = %q, want %q", class, PokeUrgentAttention) + } + // The durable Transition and the intent plan must agree that this is a + // poke: controllerTransition stamps the priority from the same + // classification the planner uses. + if escalation.InterruptionPriority == nil { + t.Error("the escalation transition records no interruption priority") + } +} + func TestInterruptionPriorityArtifactsNeverPoke(t *testing.T) { c := hsNext(t) c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} From 21a5b3094547f0a3769373c486b8b50e608700a0 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 02:42:47 +0300 Subject: [PATCH 07/31] feat(store): commit Situation history atomically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan 3 Task 5: Plan 2's fenced CommitController transaction now also persists this reconciliation's immutable Transitions, one Episode-summary version folded per Transition, one stdout-stream row per Transition, the R1 operator-artifact journaling cursor, and every notification intent — all-or-nothing, so a Situation's authoritative state and its durable history can never diverge. - ControllerCommit gains History *HistoryCommit. The controller derives it in c.commit, the single choke point all four result classes (reuse, fresh L2, fallback, blocked) already funnel through, from a new historyBasis{In, Snap, Now} threaded through commitResult/commitBlocked. Deriving in one place means no future commit path can silently skip it; a derivation failure aborts the cycle before any write. - AuthoritativeChange.Situation carries the COMMITTED projection plus this cycle's CONSUMED due reasons (never the post-commit remainder, which would make Triage materiality go dark after the first cycle). RecurrenceCount is len(PriorSituations) — the durable same-group terminal lineage Plan 2 already loads, not new counting machinery. - SnapshotInput/LoadReconciliationInput gain the minimum coherent-load fields: prior Transition, current Episode summary, Slack root publication state, latest root-sync version, last delivered root deadline, last main-channel poke, and the ordered pending artifacts. - The store re-folds the Episode summary from the row read inside the same transaction and compares the result to the committed one: migration 0017's monotonic trigger requires one write per Transition, and re-folding also proves the controller derived against current truth. - A newer pending root projection supersedes the older one in the same transaction (migration 0018 allows at most one pending root_sync per Situation), with foreign keys deferred for that one self-referencing write. - Bounded coherent read views for Slack/MCP/stdout/replay: current Episode with its source Transition, an ordered Transition page on a stable (sequence, id) cursor, exact Transition/intent by id, and the pending stdout page. Two changes outside the task's own files were required: - ProjectEpisode rejected every fold onto a terminal summary, which made R1's mandated "artifact pending when the Situation terminalizes" case underivable: each Transition of one commit carries the same captured projection (R3), so the artifact Transition already reports the closure and the terminal Transition could no longer fold. The guard now permits exactly the rest of the same terminal commit (identical terminal instant) and still rejects any later reopening. - CommitController's authoritative-attempt insert moved to a helper: the one added history call pushed it past the repo's gocyclo limit. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/situation/controller.go | 264 +++++- internal/situation/controller_test.go | 467 +++++++++++ internal/situation/history.go | 18 +- internal/situation/history_test.go | 54 ++ internal/situation/snapshot.go | 40 + internal/store/situation_controller.go | 260 +++++- internal/store/situation_history.go | 682 ++++++++++++++++ internal/store/situation_history_test.go | 993 +++++++++++++++++++++++ internal/store/situation_views.go | 193 +++++ internal/store/situation_views_test.go | 159 ++++ 10 files changed, 3075 insertions(+), 55 deletions(-) create mode 100644 internal/store/situation_history.go create mode 100644 internal/store/situation_history_test.go diff --git a/internal/situation/controller.go b/internal/situation/controller.go index db82a7a..13147de 100644 --- a/internal/situation/controller.go +++ b/internal/situation/controller.go @@ -182,6 +182,16 @@ type ControllerCommit struct { RetryAt *time.Time LastErrorClass *string Parked ParkedState + + // History is Plan 3 Task 5's addition: this reconciliation's immutable + // Transitions, the Episode summary folded across them, and every + // notification intent they warrant — derived by BuildHistoryCommit + // AFTER Plan 2 has established the authoritative state above, and + // committed by the SAME fenced CommitController transaction. nil means + // this cycle produced no durable history at all (a non-material + // reconciliation with no pending artifact and no R4 deadline refresh + // due); the commit then behaves exactly as Plan 2's did. + History *HistoryCommit } // ParkedState is CommitController's explicit instruction for the @@ -374,6 +384,20 @@ type ControllerConfig struct { // Retry bounds the transient transport-failure/rate-limit retry // schedule. Default Min=5s, Max=300s, JitterPercent=20. Retry RetryConfig + + // SlackFloor is the operator's configured minimum Interruption priority + // (config notify.slack.min_severity) that a NEW main-channel poke must + // meet. The empty value is "no floor". A poke below the floor still + // creates a durable withheld_by_operator_slack_floor intent — never an + // absent row — and the floor never suppresses a non-broadcast journal + // entry (Plan 3 Task 4, PlanNotificationIntents). + SlackFloor model.InterruptionPriority + + // RepageCooldown is how long a materially changed required action must + // wait after a delivered main-channel poke before it may create another + // one (config situations.slack.repage_cooldown_seconds). Default 900s; + // it gates exactly one poke class (PokeRequiredActionChanged). + RepageCooldown time.Duration } const ( @@ -384,6 +408,7 @@ const ( defaultControllerRetryMin = 5 * time.Second defaultControllerRetryMax = 300 * time.Second defaultControllerRetryJitterPercent = 20 + defaultControllerRepageCooldown = 900 * time.Second ) func (c ControllerConfig) withDefaults() ControllerConfig { @@ -408,6 +433,9 @@ func (c ControllerConfig) withDefaults() ControllerConfig { if c.Retry.JitterPercent <= 0 { c.Retry.JitterPercent = defaultControllerRetryJitterPercent } + if c.RepageCooldown <= 0 { + c.RepageCooldown = defaultControllerRepageCooldown + } c.Cadence = c.Cadence.withDefaults() return c } @@ -1025,17 +1053,49 @@ func (c *Controller) buildControllerState(snap Snapshot, in SnapshotInput, lc li } } -// commit calls c.store.CommitController and logs/audits the outcome — -// including a stale-claim failure, which Reconcile treats as a clean, -// expected race (spec.md: "the controller fails closed and the newer input -// remains due") rather than an unexpected error. commit_failed is a -// supplementary diagnostic event beyond spec.md's own named 13-event -// taxonomy ("Audit events cover AT LEAST" that list) — kept because a -// commit failure (most commonly a stale-claim race) is operationally -// worth its own audit trail distinct from any of the 13 named events, none -// of which name a whole-commit failure. -func (c *Controller) commit(ctx context.Context, claim Claim, commit ControllerCommit) error { - err := c.store.CommitController(ctx, claim, commit) +// historyBasis is the coherent-cycle context every commit path needs to +// derive its own durable history: LoadReconciliationInput's single read, +// this cycle's Snapshot, and the one reconciliation instant. It is threaded +// through c.commit — the single choke point all four result classes (reuse, +// fresh L2, fallback, blocked) already funnel through — precisely so +// history is derived in exactly one place and no future commit path can +// silently skip it. +type historyBasis struct { + In SnapshotInput + Snap Snapshot + Now time.Time +} + +// commit derives this reconciliation's durable history from the +// authoritative result Plan 2 just finished building, attaches it to the +// same ControllerCommit, and calls c.store.CommitController — one fenced, +// all-or-nothing transaction. It logs/audits the outcome, including a +// stale-claim failure, which Reconcile treats as a clean, expected race +// (spec.md: "the controller fails closed and the newer input remains due") +// rather than an unexpected error. commit_failed is a supplementary +// diagnostic event beyond spec.md's own named 13-event taxonomy ("Audit +// events cover AT LEAST" that list) — kept because a commit failure (most +// commonly a stale-claim race) is operationally worth its own audit trail +// distinct from any of the 13 named events, none of which name a +// whole-commit failure. +// +// A history-derivation failure aborts the cycle BEFORE any write: a +// Situation's authoritative state must never land without the history that +// state warrants, so the whole reconciliation fails closed and the +// Situation stays due. +func (c *Controller) commit(ctx context.Context, claim Claim, basis historyBasis, commit ControllerCommit) error { + history, err := c.buildHistory(claim, basis, commit) + if err != nil { + c.logger.Error("situation: controller history derivation failed", + "situation_id", claim.Situation.ID, "err", err) + c.auditAppend(ctx, "situation.controller.commit_failed", map[string]any{ + "situation_id": claim.Situation.ID, "error": err.Error(), + }) + return fmt.Errorf("situation: controller reconcile: derive history: %w", err) + } + commit.History = history + + err = c.store.CommitController(ctx, claim, commit) if err != nil { c.logger.Warn("situation: controller commit failed", "situation_id", claim.Situation.ID, "err", err) c.auditAppend(ctx, "situation.controller.commit_failed", map[string]any{ @@ -1061,6 +1121,161 @@ func (c *Controller) commit(ctx context.Context, claim Claim, commit ControllerC return nil } +// buildHistory derives this commit's immutable Transitions, the Episode +// summary folded across them, and the Slack obligations they warrant — the +// pure Plan 3 Task 4 composition, run only AFTER Plan 2 has established the +// authoritative state it reads. It returns nil when the cycle produced no +// durable history at all: a non-material reconciliation with no pending +// operator artifact and no R4 deadline refresh due. +func (c *Controller) buildHistory(claim Claim, basis historyBasis, commit ControllerCommit) (*HistoryCommit, error) { + change := authoritativeChangeOf(claim, basis, commit) + publication := PublicationInput{ + Situation: change.Situation, + PriorTransition: basis.In.PriorTransition, + RootPublished: basis.In.RootPublished, + LatestRootSyncVersion: basis.In.LatestRootSyncVersion, + LastDeliveredRootDeadlineAt: basis.In.LastDeliveredRootDeadlineAt, + LastMainChannelPokeAt: basis.In.LastMainChannelPokeAt, + SlackFloor: c.cfg.SlackFloor, + RepageCooldown: c.cfg.RepageCooldown, + Drill: change.Drill, + Now: basis.Now, + } + // R4: the root renders the committed nonterminal promise, captured here + // from the committed Operator contract — never from the Episode summary. + if !commit.Lifecycle.Terminal() { + publication.ContractDeadlineAt = commit.Assessment.ActionContract.NextUpdateAt + } + + history, err := BuildHistoryCommit(change, publication) + if err != nil { + return nil, err + } + if len(history.Transitions) == 0 && len(history.Intents) == 0 { + return nil, nil //nolint:nilnil // "this cycle warranted no history" is a legitimate, non-error result. + } + return &history, nil +} + +// authoritativeChangeOf reduces one committed reconciliation to exactly what +// deriving durable history needs. The Situation it carries is the COMMITTED +// projection — Plan 2's own lifecycle/Attention/recovery/terminal decisions +// overlaid on the coherent read — never the pre-commit row. +func authoritativeChangeOf(claim Claim, basis historyBasis, commit ControllerCommit) AuthoritativeChange { + sit := basis.In.Situation + sit.InputVersion = claim.Situation.InputVersion + sit.Lifecycle = commit.Lifecycle + sit.Attention = commit.Attention + sit.RecoveryObservedAt = commit.RecoveryObservedAt + sit.GraceUntil = commit.GraceUntil + sit.TerminalAt = commit.TerminalAt + sit.TerminalReason = commit.TerminalReason + sit.NextAssessmentAt = commit.NextAssessmentAt + sit.UpdatedAt = basis.Now + // This cycle's CONSUMED due reasons, never the remainder CommitController + // leaves behind after subtracting them: Triage materiality reads the + // consumed `triage_changed`, so handing it the post-commit remainder + // would make every Triage change after the first one invisible. + sit.DueReasons = commit.ConsumedDueReasons + + conclusion := assessmentConclusionOf(commit.Assessment) + change := AuthoritativeChange{ + Situation: sit, + Assessment: commit.Assessment, + Derivation: commit.Attempt.Derivation, + // R3: bounded projection facts captured from the coherent claim and + // this commit's own lifecycle fields — the only thing the Episode + // fold may read. + Projection: model.ProjectionFacts{ + PublicHandle: sit.PublicHandle, + EffectiveStartedAt: sit.EffectiveStartedAt, + EffectiveStartedAtBasis: sit.EffectiveStartedAtBasis, + RecoveryObservedAt: commit.RecoveryObservedAt, + GraceUntil: commit.GraceUntil, + TerminalAt: commit.TerminalAt, + TerminalReason: commit.TerminalReason, + Assessment: &conclusion, + }, + PriorTransition: basis.In.PriorTransition, + PriorSummary: basis.In.CurrentSummary, + MaterialFactHash: commit.MaterialFactHash, + EvidenceRefs: materialFactRefs(basis.Snap), + Incidents: basis.Snap.Incidents, + TriageDecisions: commit.TriageDecisions, + // spec.md: "recurrence count available from durable local Store + // facts" — this exact group's prior terminal Situations, the same + // durable lineage Plan 2 already loads for its duration + // distribution. No new counting machinery. + RecurrenceCount: len(basis.In.PriorSituations), + OperatorArtifacts: basis.In.PendingArtifacts, + Drill: situationDrill(basis.In), + Now: basis.Now, + } + switch { + case commit.Attempt.ID != "": + change.AssessmentID = stringPtrOf(commit.Attempt.ID) + case basis.In.CurrentAssessment != nil: + // A cycle that wrote no new attempt still records the Assessment + // that stays authoritative (situations.current_assessment_id is + // COALESCEd, not cleared). + change.AssessmentID = stringPtrOf(basis.In.CurrentAssessment.ID) + } + if change.Derivation == "" && basis.In.CurrentAssessment != nil { + change.Derivation = basis.In.CurrentAssessment.Derivation + } + return change +} + +// assessmentConclusionOf reduces an Assessment to the closed judgment codes +// and the bounded Sufficient-reason summary a Transition's projection may +// carry (R3) — never the full Assessment, which stays referenced by ID. +func assessmentConclusionOf(a model.Assessment) model.AssessmentConclusion { + codes := make([]string, 0, len(a.Limitations)) + for _, l := range a.Limitations { + codes = append(codes, l.Code) + } + out := model.AssessmentConclusion{ + Persistence: a.Persistence, + Impact: a.Impact, + Novelty: a.Novelty, + Causality: a.Causality, + EvidenceQuality: a.EvidenceQuality, + LimitationCodes: codes, + } + if a.SufficientReason != nil { + out.SufficientReasonCode = a.SufficientReason.Code + out.SufficientReasonSummary = a.SufficientReason.Summary + } + return out +} + +// materialFactRefs is this cycle's supporting evidence: every material +// derived fact's identity. BuildTransitions canonicalizes, deduplicates, +// and bounds the list, and merges the accepted Sufficient reason's own +// references into it. +func materialFactRefs(snap Snapshot) []string { + refs := make([]string, 0, len(snap.Facts)) + for _, f := range snap.Facts { + if f.Material { + refs = append(refs, f.ID) + } + } + return refs +} + +// situationDrill reports whether this Situation carries the Drill marker, +// using the same any-drill-delivery reduction incidentDrillParity applies +// per Incident: a disagreement fails safe toward treating the Situation as +// a Drill rather than silently publishing it as real. +func situationDrill(in SnapshotInput) bool { + for _, d := range in.Deliveries { + if d.Drill { + return true + } + } + return false +} + // commitFailedError marks an error as CommitController's own rejection (a // stale claim — lease lost or input version conflict — or a store failure // at the fenced commit), so Reconcile's span can classify it as @@ -1223,6 +1438,10 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { state := c.buildControllerState(snap, in, lc, triageDecisions, now) + // The coherent-cycle context every commit path below derives its own + // durable history from (Plan 3 Task 5). + basis := historyBasis{In: in, Snap: snap, Now: now} + base := ControllerCommit{ MaterialFactHash: snap.MaterialFactHash, AssessmentBasisHash: snap.AssessmentBasisHash, @@ -1249,7 +1468,7 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { if rr.Ok { // duration=0: a reuse commit never calls out (callID is always // nil here). - return c.commitResult(ctx, claim, base, rr.Result, nil, 0, 1, 0, now) + return c.commitResult(ctx, claim, basis, base, rr.Result, nil, 0, 1, 0) } } @@ -1273,7 +1492,7 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { // controllerParkBlocksDispatch returns false and this cycle proceeds // exactly as if not parked, naturally lifting the park. if controllerParkBlocksDispatch(in.ControllerParked, snap.MaterialFactHash) { - return c.commitBlocked(ctx, claim, base, situationID, snap, in, state, now) + return c.commitBlocked(ctx, claim, basis, base, state) } // Finding I3 ruling: a deterministic urgent floor (critical_anchor) does @@ -1360,7 +1579,7 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { // Otherwise already parked from a prior cycle on this unchanged // input: refresh the bounded projection only, touch no parked // state. - return c.commitBlocked(ctx, claim, base, situationID, snap, in, state, now) + return c.commitBlocked(ctx, claim, basis, base, state) } return fmt.Errorf("situation: controller reconcile: begin attempt: %w", err) } @@ -1374,7 +1593,7 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { } if disp.proposal != nil { result := DeriveAssessment(*disp.proposal, snap, in, state, model.DerivationModelValidated, nil, now) - return c.commitResult(ctx, claim, base, result, &disp.lastCallID, retryEpoch, workAttempt, disp.lastDuration, now) + return c.commitResult(ctx, claim, basis, base, result, &disp.lastCallID, retryEpoch, workAttempt, disp.lastDuration) } // No accepted/contradicted result: classify the last outcome (using @@ -1405,7 +1624,7 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { assessment, attempt, coverage := c.fallbackOrPreserve(situationID, snap, in, state, retryEpoch, workAttempt, now) base.Assessment, base.Attempt, base.Coverage = assessment, attempt, coverage c.finalizeCheckpoint(&base, now) - return c.commit(ctx, claim, base) + return c.commit(ctx, claim, basis, base) } // commitResult finishes building base from a successful (no-L2-needed, or @@ -1415,7 +1634,8 @@ func (c *Controller) reconcile(ctx context.Context, claim Claim) error { // when callID is non-nil (dispatchWorkBearing's own oneShot.Latency), or 0 // for a no-call commit (reuse) — threaded straight through to // buildAuthoritativeAttempt. -func (c *Controller) commitResult(ctx context.Context, claim Claim, base ControllerCommit, result AssessmentResult, callID *string, retryEpoch, workAttempt int, duration time.Duration, now time.Time) error { +func (c *Controller) commitResult(ctx context.Context, claim Claim, basis historyBasis, base ControllerCommit, result AssessmentResult, callID *string, retryEpoch, workAttempt int, duration time.Duration) error { + now := basis.Now base.Attempt = buildAuthoritativeAttempt(claim.Situation.ID, result, callID, retryEpoch, workAttempt, duration, now) base.Assessment = result.Assessment base.Coverage = result.Coverage @@ -1424,7 +1644,7 @@ func (c *Controller) commitResult(ctx context.Context, claim Claim, base Control base.LastErrorClass = nil c.finalizeCheckpoint(&base, now) - err := c.commit(ctx, claim, base) + err := c.commit(ctx, claim, basis, base) if err != nil && callID != nil { // spec.md: "A stale proposal/attempt is retained as `stale` but // changes no projection, Triage state, lifecycle, or outward @@ -1510,11 +1730,11 @@ func (c *Controller) finalizeCheckpoint(base *ControllerCommit, now time.Time) { // the zero value, so whatever was persisted stands; the one exception is // Reconcile's exhausted-but-never-parked repair, which hands in a // ParkedReasonDependency park for this commit to persist). -func (c *Controller) commitBlocked(ctx context.Context, claim Claim, base ControllerCommit, situationID string, snap Snapshot, in SnapshotInput, state ControllerState, now time.Time) error { - assessment, attempt, coverage := c.fallbackOrPreserveBlocked(situationID, snap, in, state, now) +func (c *Controller) commitBlocked(ctx context.Context, claim Claim, basis historyBasis, base ControllerCommit, state ControllerState) error { + assessment, attempt, coverage := c.fallbackOrPreserveBlocked(claim.Situation.ID, basis.Snap, basis.In, state, basis.Now) base.Assessment, base.Attempt, base.Coverage = assessment, attempt, coverage - c.finalizeCheckpoint(&base, now) - return c.commit(ctx, claim, base) + c.finalizeCheckpoint(&base, basis.Now) + return c.commit(ctx, claim, basis, base) } // fallbackOrPreserveBlocked is fallbackOrPreserve's counterpart for a cycle diff --git a/internal/situation/controller_test.go b/internal/situation/controller_test.go index f3167d8..ed4b676 100644 --- a/internal/situation/controller_test.go +++ b/internal/situation/controller_test.go @@ -1609,3 +1609,470 @@ func TestControllerReportsFinalTypedOutcomeToHealthObserverPerSituation(t *testi } }) } + +// -------------------------------------------------------------------------- +// Plan 3 Task 5: the controller derives durable history from the SAME +// authoritative result Plan 2 commits, and hands it to the store as one +// ControllerCommit. These tests exercise every result class that commits. +// -------------------------------------------------------------------------- + +// ctReuseInput returns a SnapshotInput whose current authoritative +// Assessment matches the basis the live Reconcile cycle will compute, so +// the deterministic/reuse check succeeds and no L2 call is made. +func ctReuseInput(t *testing.T) situation.SnapshotInput { + t.Helper() + in := ctBaseSnapshotInput() + in.Now = ctBaseTime.Add(10 * time.Minute) + snap := situation.BuildSnapshot(in) + in.CurrentAssessment = &situation.AuthoritativeAssessment{ + ID: "assessment-prior", SituationID: "situation-1", + AssessmentBasisHash: snap.AssessmentBasisHash, MaterialFactHash: snap.MaterialFactHash, + InputVersion: 2, Derivation: model.DerivationModelValidated, + Assessment: model.Assessment{ + SchemaVersion: model.AssessmentSchemaVersion, Persistence: model.PersistenceSustained, + Impact: model.ImpactSuspected, Novelty: model.NoveltyFamiliar, Causality: model.CausalityCorrelated, + Attention: model.AttentionObserve, Lifecycle: model.LifecycleActive, + EvidenceQuality: model.EvidenceQualityComplete, Cadence: model.CadenceSlow, + ActionContract: model.ActionContract{ + NextActor: model.NextActorNone, NextUpdateAt: &ctBaseTime, + }, + }, + } + return in +} + +// ctReconcileOnce runs one full cycle against a fake store and returns the +// single ControllerCommit it produced. +func ctReconcileOnce(t *testing.T, in situation.SnapshotInput, claim situation.Claim, + tune func(*fakeControllerStore)) situation.ControllerCommit { + t.Helper() + return ctReconcileWith(t, in, claim, &fakeAssessmentClient{}, tune) +} + +// ctReconcileWith is ctReconcileOnce with a caller-supplied provider client, +// for the work-bearing cycles a changed basis forces. +func ctReconcileWith(t *testing.T, in situation.SnapshotInput, claim situation.Claim, + client situation.AssessmentClient, tune func(*fakeControllerStore)) situation.ControllerCommit { + t.Helper() + store := &fakeControllerStore{loadInput: in} + if tune != nil { + tune(store) + } + c := ctController(t, store, client) + if err := c.Reconcile(context.Background(), claim); err != nil { + t.Fatalf("Reconcile: %v", err) + } + commits := store.snapshotCommits() + if len(commits) != 1 { + t.Fatalf("commits = %d, want 1", len(commits)) + } + return commits[0] +} + +// TestControllerHistoryFirstPublicationCommitsTransitionSummaryAndIntents +// pins the fresh-publication class: one first_authoritative_state +// Transition, its folded Episode summary, and the Slack obligations it +// warrants, all inside the ONE ControllerCommit Plan 2 already fences. +func TestControllerHistoryFirstPublicationCommitsTransitionSummaryAndIntents(t *testing.T) { + commit := ctReconcileOnce(t, ctReuseInput(t), ctBaseClaim(), nil) + + if commit.History == nil { + t.Fatal("a first authoritative state must commit durable history") + } + if len(commit.History.Transitions) != 1 { + t.Fatalf("transitions = %d, want 1: %+v", len(commit.History.Transitions), commit.History.Transitions) + } + tr := commit.History.Transitions[0] + if tr.Reason != model.ReasonFirstAuthoritativeState { + t.Fatalf("transition reason = %q, want first_authoritative_state", tr.Reason) + } + if tr.Sequence != 1 { + t.Fatalf("transition sequence = %d, want 1", tr.Sequence) + } + if tr.SituationID != "situation-1" || tr.InputVersion != 3 { + t.Fatalf("transition identity = (%q,%d), want (situation-1,3)", tr.SituationID, tr.InputVersion) + } + if tr.AssessmentID == nil || *tr.AssessmentID != commit.Attempt.ID { + t.Fatalf("transition assessment id = %v, want this commit's own attempt %q", tr.AssessmentID, commit.Attempt.ID) + } + if tr.Lifecycle != commit.Lifecycle || tr.Attention != commit.Attention { + t.Fatalf("transition state (%q,%q) does not match the committed projection (%q,%q)", + tr.Lifecycle, tr.Attention, commit.Lifecycle, commit.Attention) + } + if commit.History.Summary == nil || commit.History.Summary.Version != 1 { + t.Fatalf("episode summary = %+v, want version 1", commit.History.Summary) + } + if commit.History.Summary.SourceTransitionSequence != tr.Sequence { + t.Fatalf("summary source sequence = %d, want %d", commit.History.Summary.SourceTransitionSequence, tr.Sequence) + } + root := 0 + for _, intent := range commit.History.Intents { + if intent.EffectClass == model.EffectRootSync { + root++ + if !intent.MainChannelPoke { + t.Fatal("an unpublished root's first projection IS the main-channel poke") + } + } + } + if root != 1 { + t.Fatalf("root_sync intents = %d, want exactly 1", root) + } +} + +// TestControllerHistoryRevalidatedReuseCommitsNoHistory pins R4: a reuse +// cycle writes a new Assessment with new IDs and a refreshed next_update_at +// and still creates no Transition, no Episode version, and no Slack intent. +func TestControllerHistoryRevalidatedReuseCommitsNoHistory(t *testing.T) { + in := ctReuseInput(t) + first := ctReconcileOnce(t, in, ctBaseClaim(), nil) + + in.PriorTransition = &first.History.Transitions[0] + in.CurrentSummary = first.History.Summary + claim := ctBaseClaim() + claim.Situation.DueReasons = nil + + second := ctReconcileOnce(t, in, claim, nil) + if second.Attempt.Derivation != model.DerivationRevalidatedReuse { + t.Fatalf("second cycle derivation = %q, want revalidated_reuse", second.Attempt.Derivation) + } + if second.History != nil { + t.Fatalf("a revalidated_reuse cycle must commit no history, got %+v", second.History) + } +} + +// TestControllerHistoryDeadlineRefreshIsTheOnlyNonMaterialSlackEffect pins +// R4's one narrowing: a published root whose delivered promise has passed +// gets exactly one coalescible root_sync refresh — no Transition, no +// journal entry, no poke. +func TestControllerHistoryDeadlineRefreshIsTheOnlyNonMaterialSlackEffect(t *testing.T) { + in := ctReuseInput(t) + first := ctReconcileOnce(t, in, ctBaseClaim(), nil) + + expired := ctBaseTime.Add(-time.Hour) + in.PriorTransition = &first.History.Transitions[0] + in.CurrentSummary = first.History.Summary + in.RootPublished = true + in.LastDeliveredRootDeadlineAt = &expired + claim := ctBaseClaim() + claim.Situation.DueReasons = nil + + second := ctReconcileOnce(t, in, claim, nil) + if second.History == nil { + t.Fatal("a due deadline refresh must still commit history") + } + if len(second.History.Transitions) != 0 { + t.Fatalf("a deadline refresh creates no Transition, got %+v", second.History.Transitions) + } + if second.History.Summary != nil { + t.Fatalf("a deadline refresh creates no Episode version, got %+v", second.History.Summary) + } + if len(second.History.Intents) != 1 { + t.Fatalf("deadline-refresh intents = %d, want exactly 1: %+v", len(second.History.Intents), second.History.Intents) + } + refresh := second.History.Intents[0] + if refresh.EffectClass != model.EffectRootSync { + t.Fatalf("refresh effect class = %q, want root_sync", refresh.EffectClass) + } + if refresh.MainChannelPoke { + t.Fatal("a deadline refresh is a silent edit, never a poke") + } + if refresh.ContractDeadlineAt == nil || refresh.ContractDeadlineAt.Equal(expired) { + t.Fatalf("refresh contract deadline = %v, want the newly committed one", refresh.ContractDeadlineAt) + } + if refresh.TransitionID == nil || *refresh.TransitionID != first.History.Transitions[0].ID { + t.Fatalf("refresh authority = %v, want the prior Transition %q", refresh.TransitionID, first.History.Transitions[0].ID) + } +} + +// TestControllerHistoryJournalsPendingArtifactsBeforeControllerState pins +// R1: every applied-and-unjournaled artifact is journaled in this one +// commit, in the loaded order, before any controller-state Transition. +func TestControllerHistoryJournalsPendingArtifactsBeforeControllerState(t *testing.T) { + in := ctReuseInput(t) + in.PendingArtifacts = []situation.OperatorArtifactInput{ + { + InputID: "input-1", Kind: "operator_annotation_recorded", + AnnotationID: stringPtrForTest("11"), AppliedInputVersion: 3, + OccurredAt: ctBaseTime.Add(time.Minute), AttributedActor: "", + Headline: "Operator note recorded (observation)", Detail: "Checked the deploy log.", + }, + { + InputID: "input-2", Kind: "captured_verdict_recorded", + VerdictID: stringPtrForTest("22"), AppliedInputVersion: 3, + OccurredAt: ctBaseTime.Add(2 * time.Minute), AttributedActor: "human", + Headline: "Captured verdict recorded (confirmation)", Detail: "", + }, + } + + commit := ctReconcileOnce(t, in, ctBaseClaim(), nil) + if commit.History == nil || len(commit.History.Transitions) != 3 { + t.Fatalf("transitions = %+v, want two artifacts then the controller state", commit.History) + } + for i, want := range []string{"input-1", "input-2"} { + tr := commit.History.Transitions[i] + if tr.Reason != model.ReasonOperatorArtifactRecorded { + t.Fatalf("transition %d reason = %q, want operator_artifact_recorded", i, tr.Reason) + } + if tr.OperatorArtifactInputID == nil || *tr.OperatorArtifactInputID != want { + t.Fatalf("transition %d artifact input = %v, want %q", i, tr.OperatorArtifactInputID, want) + } + if tr.Actor != model.ActorAttributedOperator { + t.Fatalf("transition %d actor = %q, want attributed_operator", i, tr.Actor) + } + if tr.Sequence != i+1 { + t.Fatalf("transition %d sequence = %d, want %d", i, tr.Sequence, i+1) + } + } + last := commit.History.Transitions[2] + if last.Reason != model.ReasonFirstAuthoritativeState || last.Sequence != 3 { + t.Fatalf("last transition = (%q,%d), want (first_authoritative_state,3)", last.Reason, last.Sequence) + } + if commit.History.Summary == nil || commit.History.Summary.Version != 3 { + t.Fatalf("episode summary = %+v, want one version per Transition (3)", commit.History.Summary) + } +} + +// TestControllerHistoryUsesThisCycleConsumedDueReasons is the two-cycle +// regression the artifact/Triage materiality rule depends on: history must +// be derived against the due reasons THIS claim consumed, never against the +// remainder Plan 2's own commit leaves behind. Cycle one consumes +// triage_changed and must record a triage_state_changed Transition on an +// otherwise unchanged tuple; cycle two, with the reason consumed, must +// record nothing. +func TestControllerHistoryUsesThisCycleConsumedDueReasons(t *testing.T) { + in := ctReuseInput(t) + first := ctReconcileOnce(t, in, ctBaseClaim(), nil) + + in.PriorTransition = &first.History.Transitions[0] + in.CurrentSummary = first.History.Summary + + consuming := ctBaseClaim() + consuming.Situation.DueReasons = []model.DueReason{model.DueTriageChanged} + second := ctReconcileOnce(t, in, consuming, nil) + if second.History == nil || len(second.History.Transitions) != 1 { + t.Fatalf("a consumed triage_changed on an unchanged tuple must record one Transition, got %+v", second.History) + } + if got := second.History.Transitions[0].Reason; got != model.ReasonTriageStateChanged { + t.Fatalf("transition reason = %q, want triage_state_changed", got) + } + + in.PriorTransition = &second.History.Transitions[0] + in.CurrentSummary = second.History.Summary + settled := ctBaseClaim() + settled.Situation.DueReasons = nil + third := ctReconcileOnce(t, in, settled, nil) + if third.History != nil { + t.Fatalf("the next cycle must not re-record a consumed triage change, got %+v", third.History) + } +} + +// TestControllerHistoryRecurrenceCountComesFromPriorTerminalSituations pins +// where the Episode summary's recurrence count comes from: durable local +// Store facts — this exact group's prior terminal Situations — not a new +// counter. +func TestControllerHistoryRecurrenceCountComesFromPriorTerminalSituations(t *testing.T) { + in := ctReuseInput(t) + in.PriorSituations = []situation.CompletedSituation{ + {ID: "prior-a", GroupKey: "group-1", EffectiveStartedAt: ctBaseTime.Add(-48 * time.Hour), TerminalAt: ctBaseTime.Add(-47 * time.Hour), TerminalReason: model.TerminalReasonObservationDeadline}, + {ID: "prior-b", GroupKey: "group-1", EffectiveStartedAt: ctBaseTime.Add(-24 * time.Hour), TerminalAt: ctBaseTime.Add(-23 * time.Hour), TerminalReason: model.TerminalReasonObservationDeadline}, + } + + commit := ctReconcileOnce(t, in, ctBaseClaim(), nil) + if commit.History == nil || commit.History.Summary == nil { + t.Fatalf("expected history, got %+v", commit.History) + } + if got := commit.History.Summary.RecurrenceCount; got != 2 { + t.Fatalf("summary recurrence count = %d, want 2 (the prior terminal Situations in this group)", got) + } + if got := commit.History.Transitions[0].Journal.RecurrenceCount; got != 2 { + t.Fatalf("transition journal recurrence count = %d, want 2", got) + } +} + +// TestControllerHistoryBlockedCycleStillCommitsHistory covers the blocked +// result class: a cycle that may not dispatch further L2 work still +// establishes authoritative state, so it still records the history that +// state warrants. +func TestControllerHistoryBlockedCycleStillCommitsHistory(t *testing.T) { + in := ctBaseSnapshotInput() + in.Now = ctBaseTime.Add(10 * time.Minute) + in.ControllerParked = situation.ControllerParkedState{ + At: &ctBaseTime, Reason: situation.ParkedReasonDependency, + MaterialFactHash: situation.BuildSnapshot(in).MaterialFactHash, + } + + commit := ctReconcileOnce(t, in, ctBaseClaim(), func(f *fakeControllerStore) { + f.beginErr = situation.ErrControllerAttemptsExhausted + }) + if commit.History == nil || len(commit.History.Transitions) != 1 { + t.Fatalf("a blocked cycle that establishes first authoritative state must record it, got %+v", commit.History) + } + if got := commit.History.Transitions[0].Reason; got != model.ReasonFirstAuthoritativeState { + t.Fatalf("blocked-cycle transition reason = %q, want first_authoritative_state", got) + } +} + +// TestControllerHistoryFallbackCycleCommitsHistory covers the fallback +// result class: L2 failed, a deterministic fallback Assessment became +// authoritative, and that IS the Situation's first durable state. +func TestControllerHistoryFallbackCycleCommitsHistory(t *testing.T) { + in := ctBaseSnapshotInput() + in.Now = ctBaseTime.Add(10 * time.Minute) + store := &fakeControllerStore{loadInput: in, beginWorkAttempt: 1} + client := &fakeAssessmentClient{responses: []func() (llm.OneShotCompletion, error){ + func() (llm.OneShotCompletion, error) { + return llm.OneShotCompletion{RequestStarted: llm.RequestStartStatusFalse}, errors.New("provider unreachable") + }, + }} + c := ctController(t, store, client) + if err := c.Reconcile(context.Background(), ctBaseClaim()); err != nil { + t.Fatalf("Reconcile: %v", err) + } + commits := store.snapshotCommits() + if len(commits) != 1 { + t.Fatalf("commits = %d, want 1", len(commits)) + } + commit := commits[0] + if commit.Attempt.Derivation != model.DerivationDeterministicFallback { + t.Fatalf("derivation = %q, want deterministic_fallback", commit.Attempt.Derivation) + } + if commit.History == nil || len(commit.History.Transitions) != 1 { + t.Fatalf("a fallback cycle's first authoritative state must be recorded, got %+v", commit.History) + } + if got := commit.History.Transitions[0].Actor; got != model.ActorDeterministicController { + t.Fatalf("fallback transition actor = %q, want deterministic_controller", got) + } +} + +// TestControllerHistoryCrashBeforeCommitCreatesNothing pins the crash +// boundary: when the fenced commit itself is rejected, the cycle reports +// failure and no durable history exists — the store never partially +// applies a ControllerCommit. +func TestControllerHistoryCrashBeforeCommitCreatesNothing(t *testing.T) { + store := &fakeControllerStore{loadInput: ctReuseInput(t), commitErr: model.ErrSituationLeaseLost} + c := ctController(t, store, &fakeAssessmentClient{}) + if err := c.Reconcile(context.Background(), ctBaseClaim()); err == nil { + t.Fatal("Reconcile with a rejected commit must report the failure") + } + commits := store.snapshotCommits() + if len(commits) != 1 { + t.Fatalf("commit attempts = %d, want exactly 1 (never retried in-cycle)", len(commits)) + } + if commits[0].History == nil { + t.Fatal("the rejected commit must still have carried its derived history: the store, not the controller, decides atomicity") + } +} + +func stringPtrForTest(s string) *string { return &s } + +// ctResolvedInput is ctReuseInput with every member delivery resolved — the +// durable shape that moves an active Situation to recovery_pending. +func ctResolvedInput(t *testing.T) situation.SnapshotInput { + t.Helper() + in := ctReuseInput(t) + in.Deliveries = []situation.Delivery{ctDelivery("delivery-1", "incident-1", false, "warning")} + return in +} + +// TestControllerHistoryRecordsRecoveryThenRefire walks the lifecycle paths a +// real Situation takes through three cycles: first publication, recovery +// observation, and a refire — each a separate immutable Transition on the +// same Episode. +func TestControllerHistoryRecordsRecoveryThenRefire(t *testing.T) { + in := ctReuseInput(t) + first := ctReconcileOnce(t, in, ctBaseClaim(), nil) + + // Cycle 2: the source resolved. The basis changed, so this is a + // work-bearing cycle; the provider is unavailable and the deterministic + // path still establishes recovery_pending. + recovering := ctResolvedInput(t) + recovering.PriorTransition = &first.History.Transitions[0] + recovering.CurrentSummary = first.History.Summary + second := ctReconcileWith(t, recovering, ctBaseClaim(), &fakeAssessmentClient{}, func(f *fakeControllerStore) { + f.beginWorkAttempt = 1 + }) + if second.Lifecycle != model.LifecycleRecoveryPending { + t.Fatalf("cycle 2 lifecycle = %q, want recovery_pending", second.Lifecycle) + } + if second.History == nil || len(second.History.Transitions) != 1 { + t.Fatalf("cycle 2 history = %+v, want one Transition", second.History) + } + if got := second.History.Transitions[0].Reason; got != model.ReasonRecoveryObserved { + t.Fatalf("cycle 2 transition reason = %q, want recovery_observed", got) + } + if second.History.Summary.Version != 2 { + t.Fatalf("cycle 2 summary version = %d, want 2", second.History.Summary.Version) + } + + // Cycle 3: it fired again before the grace deadline. + refiring := ctReuseInput(t) + refiring.Situation.Lifecycle = model.LifecycleRecoveryPending + refiring.Situation.RecoveryObservedAt = &ctBaseTime + graceUntil := ctBaseTime.Add(time.Hour) + refiring.Situation.GraceUntil = &graceUntil + refiring.PriorTransition = &second.History.Transitions[0] + refiring.CurrentSummary = second.History.Summary + third := ctReconcileWith(t, refiring, ctBaseClaim(), &fakeAssessmentClient{}, func(f *fakeControllerStore) { + f.beginWorkAttempt = 1 + }) + if third.Lifecycle != model.LifecycleActive { + t.Fatalf("cycle 3 lifecycle = %q, want active", third.Lifecycle) + } + if third.History == nil || len(third.History.Transitions) != 1 { + t.Fatalf("cycle 3 history = %+v, want one Transition", third.History) + } + if got := third.History.Transitions[0].Reason; got != model.ReasonRecoveryFailed { + t.Fatalf("cycle 3 transition reason = %q, want recovery_failed", got) + } + if got := third.History.Transitions[0].Sequence; got != 3 { + t.Fatalf("cycle 3 transition sequence = %d, want 3", got) + } +} + +// TestControllerHistoryTerminalCycleJournalsPendingArtifactFirst pins R1's +// terminal ordering: an artifact still pending when the Situation +// terminalizes is journaled in the same commit, BEFORE the terminal +// Transition — a terminal Episode never reopens to absorb it later. +func TestControllerHistoryTerminalCycleJournalsPendingArtifactFirst(t *testing.T) { + in := ctReuseInput(t) + first := ctReconcileOnce(t, in, ctBaseClaim(), nil) + + closing := ctResolvedInput(t) + closing.Situation.Lifecycle = model.LifecycleRecoveryPending + closing.Situation.RecoveryObservedAt = &ctBaseTime + expiredGrace := ctBaseTime.Add(time.Minute) + closing.Situation.GraceUntil = &expiredGrace + closing.PriorTransition = &first.History.Transitions[0] + closing.CurrentSummary = first.History.Summary + closing.PendingArtifacts = []situation.OperatorArtifactInput{{ + InputID: "input-late", Kind: "operator_annotation_recorded", + AnnotationID: stringPtrForTest("77"), AppliedInputVersion: 3, + OccurredAt: ctBaseTime.Add(time.Minute), Headline: "Operator note recorded (observation)", + }} + + commit := ctReconcileWith(t, closing, ctBaseClaim(), &fakeAssessmentClient{}, func(f *fakeControllerStore) { + f.beginWorkAttempt = 1 + }) + if commit.Lifecycle != model.LifecycleRecovered { + t.Fatalf("lifecycle = %q, want recovered", commit.Lifecycle) + } + if commit.History == nil || len(commit.History.Transitions) != 2 { + t.Fatalf("history = %+v, want the artifact then the terminal Transition", commit.History) + } + artifact, terminal := commit.History.Transitions[0], commit.History.Transitions[1] + if artifact.Reason != model.ReasonOperatorArtifactRecorded || artifact.Sequence != 2 { + t.Fatalf("first transition = (%q,%d), want (operator_artifact_recorded,2)", artifact.Reason, artifact.Sequence) + } + if terminal.Reason != model.ReasonRecovered || terminal.Sequence != 3 { + t.Fatalf("second transition = (%q,%d), want (recovered,3)", terminal.Reason, terminal.Sequence) + } + if terminal.Projection.TerminalAt == nil { + t.Fatal("a terminal Transition must carry the committed terminal instant") + } + if commit.History.Summary == nil || commit.History.Summary.TerminalAt == nil { + t.Fatalf("terminal Episode summary = %+v, want a terminal instant", commit.History.Summary) + } + if commit.History.Summary.DurationSeconds == nil { + t.Fatal("a terminal Episode summary must carry its derived duration") + } +} diff --git a/internal/situation/history.go b/internal/situation/history.go index 321a3ed..fef3dbf 100644 --- a/internal/situation/history.go +++ b/internal/situation/history.go @@ -854,15 +854,29 @@ func ProjectEpisode(prior *model.EpisodeSummary, t model.Transition) (model.Epis // validateFold rejects every incoherent fold: a Transition from another // Situation, one that skips or repeats a sequence, one that moves time -// backwards, and any fold at all onto a terminal Episode — a later firing +// backwards, and any fold onto an already-terminal Episode — a later firing // creates a separately linked Situation through Plan 1/2 recurrence // ownership. +// +// The one permitted fold onto a terminal summary is the rest of the SAME +// terminal commit. R1 journals every pending operator artifact before the +// controller-state Transition, and every Transition of one commit carries +// the same captured projection (R3) — so a commit that both journals an +// artifact and closes the Situation folds an artifact Transition that +// already reports the closure, then the terminal Transition itself. A +// Transition reporting the identical terminal instant is by construction +// part of the commit that closed the Episode, never a later reopening: a +// terminal Situation is never claimed again (ClaimDueSituations selects +// only active/recovery_pending) and a later artifact is recorded, never +// journaled (R2). func validateFold(prior model.EpisodeSummary, t model.Transition) error { + sameTerminalCommit := prior.TerminalAt != nil && t.Projection.TerminalAt != nil && + t.Projection.TerminalAt.Equal(*prior.TerminalAt) switch { case prior.SituationID != t.SituationID: return fmt.Errorf("situation: project episode: transition belongs to situation %q, summary to %q", t.SituationID, prior.SituationID) - case prior.TerminalAt != nil: + case prior.TerminalAt != nil && !sameTerminalCommit: return fmt.Errorf("situation: project episode: episode is terminal at %s and never reopens", prior.TerminalAt) case t.Sequence != prior.SourceTransitionSequence+1: return fmt.Errorf("situation: project episode: transition sequence %d is not contiguous with summary sequence %d", diff --git a/internal/situation/history_test.go b/internal/situation/history_test.go index 7ca9a97..69f4279 100644 --- a/internal/situation/history_test.go +++ b/internal/situation/history_test.go @@ -1450,3 +1450,57 @@ func TestBuildTransitionsMarksDrills(t *testing.T) { } } } + +// TestProjectEpisodeFoldsTheRestOfOneTerminalCommitButNeverReopens pins the +// one permitted fold onto a terminal summary (Plan 3 Task 5 integration): a +// commit that journals a pending artifact AND closes the Situation folds +// both of its Transitions, because every Transition of one commit carries +// the same terminal instant (R1/R3). A Transition reporting a DIFFERENT +// terminal instant — or none — still never reopens a closed Episode. +func TestProjectEpisodeFoldsTheRestOfOneTerminalCommitButNeverReopens(t *testing.T) { + c := hsNext(t) + hsClosedUnknown(&c) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-terminal", artifactKindAnnotation, c.Now)} + + commit, err := BuildHistoryCommit(c, PublicationInput{Situation: c.Situation, Now: c.Now}) + if err != nil { + t.Fatalf("BuildHistoryCommit: %v", err) + } + if len(commit.Transitions) != 2 { + t.Fatalf("transitions = %d, want the artifact then the terminal state", len(commit.Transitions)) + } + if commit.Summary == nil { + t.Fatal("a terminal commit that journals an artifact must still fold a summary") + } + if commit.Summary.Version != c.PriorSummary.Version+2 { + t.Fatalf("summary version = %d, want one fold per Transition (%d)", commit.Summary.Version, c.PriorSummary.Version+2) + } + if commit.Summary.SourceTransitionSequence != commit.Transitions[1].Sequence { + t.Fatalf("summary source sequence = %d, want the terminal Transition's %d", + commit.Summary.SourceTransitionSequence, commit.Transitions[1].Sequence) + } + if len(commit.Summary.RecordedOperatorContext) != 1 { + t.Fatalf("recorded operator context = %v, want the journaled artifact", commit.Summary.RecordedOperatorContext) + } + + // A later Transition reporting a different closure never reopens it. + later := commit.Transitions[1] + later.Sequence++ + later.CreatedAt = later.CreatedAt.Add(time.Hour) + shifted := later.CreatedAt + later.Projection.TerminalAt = &shifted + if _, err := ProjectEpisode(commit.Summary, later); err == nil { + t.Fatal("a Transition from a different closure must never fold onto a terminal Episode") + } + + // So does one that reports no closure at all. + reopening := commit.Transitions[1] + reopening.Sequence++ + reopening.Projection.TerminalAt = nil + reopening.Projection.TerminalReason = nil + reopening.Lifecycle = model.LifecycleActive + reopening.ActionContract = hsRunningTriageContract(c.Now.Add(time.Minute)) + if _, err := ProjectEpisode(commit.Summary, reopening); err == nil { + t.Fatal("a nonterminal Transition must never reopen a terminal Episode") + } +} diff --git a/internal/situation/snapshot.go b/internal/situation/snapshot.go index eef8765..ae3b055 100644 --- a/internal/situation/snapshot.go +++ b/internal/situation/snapshot.go @@ -35,6 +35,46 @@ type SnapshotInput struct { // L2 work (Finding I1 — spec.md: "Policy rejection, unsupported scope, // and unsupported capability are permanent for the unchanged basis"). ControllerParked ControllerParkedState + + // ---------------------------------------------------------------- + // Plan 3 Task 5: the minimum coherent history/delivery context one + // reconciliation needs to derive its own durable history, all read + // inside LoadReconciliationInput's single read transaction so a + // caller can never combine a newer Episode summary with an + // unavailable source Transition. + // ---------------------------------------------------------------- + + // PriorTransition is the Situation's current Transition (situations. + // current_transition_id) before this cycle commits, or nil when no + // Transition exists yet — a fresh Plan 3 Situation, or a Plan 1/2 + // Situation that predates migration 0017. + PriorTransition *model.Transition + // CurrentSummary is the Situation's one current Episode-summary + // projection, or nil when no Transition has ever been folded. It is + // always coherent with PriorTransition: both come from the same read + // transaction. + CurrentSummary *model.EpisodeSummary + // RootPublished reports whether the Situation's Slack root has durable + // coordinates (situations.slack_channel/slack_root_ts, migration 0018). + // An unpublished root's first post IS the main-channel poke. + RootPublished bool + // LatestRootSyncVersion is the highest Episode-summary version any + // root_sync intent for this Situation already renders, or nil when + // none exists — the guard that keeps a commit from planning a root + // older than one already queued or delivered. + LatestRootSyncVersion *int + // LastDeliveredRootDeadlineAt is the promised-update instant the most + // recently DELIVERED root_sync actually put on screen (R4). A refresh + // is due only once this promise has passed and the committed contract + // carries a different one. + LastDeliveredRootDeadlineAt *time.Time + // LastMainChannelPokeAt is when this Situation last delivered a + // main-channel poke — the basis for the configured repage cooldown. + LastMainChannelPokeAt *time.Time + // PendingArtifacts is every applied-and-unjournaled durable operator + // artifact input for this Situation, ordered by (applied_input_version, + // occurred_at, id) exactly as R1 requires. Empty on an ordinary cycle. + PendingArtifacts []OperatorArtifactInput } // ControllerParkedState is SnapshotInput's own read of the Situation's diff --git a/internal/store/situation_controller.go b/internal/store/situation_controller.go index bb65751..60370af 100644 --- a/internal/store/situation_controller.go +++ b/internal/store/situation_controller.go @@ -8,8 +8,10 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "strings" "time" + "unicode/utf8" "github.com/google/uuid" @@ -111,21 +113,197 @@ func (s *Store) LoadReconciliationInput(ctx context.Context, claim situation.Cla return situation.SnapshotInput{}, err } + // Plan 3 Task 5: the durable history/delivery context this cycle needs + // to derive its own history, read inside this same transaction so the + // prior Transition, the current Episode summary, and the pending + // artifacts can never be combined across two different snapshots. + priorTransition, err := loadCurrentTransitionTx(ctx, tx, sit.ID) + if err != nil { + return situation.SnapshotInput{}, err + } + currentSummary, err := loadEpisodeSummaryTx(ctx, tx, sit.ID) + if err != nil { + return situation.SnapshotInput{}, err + } + publication, err := loadPublicationContextTx(ctx, tx, sit.ID) + if err != nil { + return situation.SnapshotInput{}, err + } + artifacts, err := loadPendingOperatorArtifactsTx(ctx, tx, sit.ID) + if err != nil { + return situation.SnapshotInput{}, err + } + if err := tx.Commit(); err != nil { return situation.SnapshotInput{}, fmt.Errorf("store: commit load reconciliation input: %w", err) } return situation.SnapshotInput{ - Situation: sit, - Deliveries: deliveries, - Incidents: incidents, - PriorSituations: prior, - CurrentAssessment: current, - Now: now.UTC(), - ControllerParked: parked, + Situation: sit, + Deliveries: deliveries, + Incidents: incidents, + PriorSituations: prior, + CurrentAssessment: current, + Now: now.UTC(), + ControllerParked: parked, + PriorTransition: priorTransition, + CurrentSummary: currentSummary, + RootPublished: publication.rootPublished, + LatestRootSyncVersion: publication.latestRootSyncVersion, + LastDeliveredRootDeadlineAt: publication.lastDeliveredRootDeadlineAt, + LastMainChannelPokeAt: publication.lastMainChannelPokeAt, + PendingArtifacts: artifacts, }, nil } +// publicationContext is the Slack-delivery side of one coherent load: what +// is already on screen for this Situation and what promise it currently +// makes (R4). +type publicationContext struct { + rootPublished bool + latestRootSyncVersion *int + lastDeliveredRootDeadlineAt *time.Time + lastMainChannelPokeAt *time.Time +} + +// loadPublicationContextTx reads the Situation's durable Slack root +// coordinates (migration 0018) plus the three delivery facts publication +// planning needs: the newest Episode-summary version any root projection +// already renders, the promised-update instant the last DELIVERED root +// actually put on screen, and when this Situation last poked the main +// channel. +func loadPublicationContextTx(ctx context.Context, tx *sql.Tx, situationID string) (publicationContext, error) { + var out publicationContext + var channel, rootTS sql.NullString + err := tx.QueryRowContext(ctx, `SELECT slack_channel, slack_root_ts FROM situations WHERE id = ?`, situationID). + Scan(&channel, &rootTS) + if errors.Is(err, sql.ErrNoRows) { + return publicationContext{}, ErrNotFound + } + if err != nil { + return publicationContext{}, fmt.Errorf("store: read situation slack root: %w", err) + } + out.rootPublished = channel.Valid && rootTS.Valid + + var latestVersion sql.NullInt64 + if err := tx.QueryRowContext(ctx, ` + SELECT MAX(summary_version) FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync'`, situationID).Scan(&latestVersion); err != nil { + return publicationContext{}, fmt.Errorf("store: read latest root sync version: %w", err) + } + if latestVersion.Valid { + v := int(latestVersion.Int64) + out.latestRootSyncVersion = &v + } + + var deadline sql.NullString + err = tx.QueryRowContext(ctx, ` + SELECT contract_deadline_at FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'delivered' + ORDER BY delivered_at DESC, id DESC LIMIT 1`, situationID).Scan(&deadline) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return publicationContext{}, fmt.Errorf("store: read last delivered root deadline: %w", err) + } + if out.lastDeliveredRootDeadlineAt, err = timePtr(deadline); err != nil { + return publicationContext{}, err + } + + var poke sql.NullString + if err := tx.QueryRowContext(ctx, ` + SELECT MAX(delivered_at) FROM notification_intents + WHERE situation_id = ? AND main_channel_poke = 1 AND status = 'delivered'`, situationID).Scan(&poke); err != nil { + return publicationContext{}, fmt.Errorf("store: read last main channel poke: %w", err) + } + if out.lastMainChannelPokeAt, err = timePtr(poke); err != nil { + return publicationContext{}, err + } + return out, nil +} + +// loadPendingOperatorArtifactsTx reads every applied-and-unjournaled +// operator artifact input this Situation owns, in R1's exact +// (applied_input_version, occurred_at, id) order, with the bounded content +// its journal entry renders read from the durable artifact itself. An +// annotation carries no attributed actor column of its own (migration +// 0010), so only a Captured verdict's label provenance is attributed here. +func loadPendingOperatorArtifactsTx(ctx context.Context, tx *sql.Tx, situationID string) ([]situation.OperatorArtifactInput, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT o.id, o.kind, o.annotation_id, o.verdict_id, + COALESCE(o.applied_input_version, 0), o.occurred_at, + a.kind, a.note, v.verdict, v.source, v.cause_category + FROM situation_input_outbox o + LEFT JOIN incident_annotations a ON a.id = o.annotation_id + LEFT JOIN incident_verdicts v ON v.id = o.verdict_id + WHERE o.applied_situation_id = ? AND o.journal_state = 'pending' + ORDER BY o.applied_input_version ASC, o.occurred_at ASC, o.id ASC`, situationID) + if err != nil { + return nil, fmt.Errorf("store: load pending operator artifacts: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []situation.OperatorArtifactInput{} + for rows.Next() { + var a situation.OperatorArtifactInput + var annotationID, verdictID sql.NullInt64 + var occurredAt string + var annotationKind, annotationNote, verdict, source, causeCategory sql.NullString + if err := rows.Scan(&a.InputID, &a.Kind, &annotationID, &verdictID, &a.AppliedInputVersion, &occurredAt, + &annotationKind, &annotationNote, &verdict, &source, &causeCategory); err != nil { + return nil, fmt.Errorf("store: scan pending operator artifact: %w", err) + } + if a.OccurredAt, err = time.Parse(time.RFC3339Nano, occurredAt); err != nil { + return nil, fmt.Errorf("store: parse operator artifact %s occurred_at: %w", a.InputID, err) + } + a.OccurredAt = a.OccurredAt.UTC() + switch { + case annotationID.Valid: + id := strconv.FormatInt(annotationID.Int64, 10) + a.AnnotationID = &id + a.Headline = boundedArtifactText("Operator note recorded"+parenthetical(annotationKind), maxArtifactHeadline) + a.Detail = boundedArtifactText(annotationNote.String, maxArtifactDetail) + case verdictID.Valid: + id := strconv.FormatInt(verdictID.Int64, 10) + a.VerdictID = &id + a.Headline = boundedArtifactText("Captured verdict recorded"+parenthetical(verdict), maxArtifactHeadline) + a.Detail = boundedArtifactText(causeCategory.String, maxArtifactDetail) + a.AttributedActor = boundedArtifactText(source.String, maxArtifactActor) + } + out = append(out, a) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate pending operator artifacts: %w", err) + } + return out, nil +} + +// Bounds mirroring internal/situation's own Transition/journal bounds, so a +// value read here can never be rejected for length by model validation. +const ( + maxArtifactHeadline = 200 + maxArtifactDetail = 2000 + maxArtifactActor = 200 +) + +func parenthetical(v sql.NullString) string { + if !v.Valid || strings.TrimSpace(v.String) == "" { + return "" + } + return " (" + v.String + ")" +} + +// boundedArtifactText truncates s to at most limit bytes without splitting a +// rune. +func boundedArtifactText(s string, limit int) string { + if len(s) <= limit { + return s + } + cut := s[:limit] + for len(cut) > 0 && !utf8.ValidString(cut) { + cut = cut[:len(cut)-1] + } + return cut +} + // readControllerParkedStateTx reads situationID's current controller_parked_at/ // controller_parked_reason/current_material_fact_hash columns directly — raw // ALTER TABLE columns (migration 0015) Plan 1's model.Situation carries no Go @@ -1713,30 +1891,9 @@ func (s *Store) CommitController(ctx context.Context, claim situation.Claim, com } // 1. Insert the new authoritative attempt (if any) and its coverage. - var newAssessmentID sql.NullString - if commit.Attempt.ID != "" { - seq, err := nextAssessmentAttemptSequenceTx(ctx, tx, claim.Situation.ID) - if err != nil { - return err - } - attempt := commit.Attempt - attempt.Sequence = seq - p, err := prepareAssessmentAttempt(attempt, commit.MaterialFactHash) - if err != nil { - return err - } - if err := insertAssessmentAttemptTx(ctx, tx, p); err != nil { - return err - } - for _, cov := range commit.Coverage { - if _, err := tx.ExecContext(ctx, ` - INSERT INTO situation_assessment_coverage (assessment_attempt_id, incident_id, membership_digest, incident_input_digest) - VALUES (?, ?, ?, ?) ON CONFLICT(assessment_attempt_id, incident_id) DO NOTHING`, - attempt.ID, cov.IncidentID, cov.MembershipDigest, cov.IncidentInputDigest); err != nil { - return fmt.Errorf("store: insert assessment coverage: %w", err) - } - } - newAssessmentID = sql.NullString{String: attempt.ID, Valid: true} + newAssessmentID, err := commitAuthoritativeAttemptTx(ctx, tx, claim.Situation.ID, commit) + if err != nil { + return err } // 2. Apply Triage decisions sharing this same commit. @@ -1844,12 +2001,53 @@ func (s *Store) CommitController(ctx context.Context, claim situation.Claim, com return situationmodel.ErrSituationLeaseLost } + // 7. Plan 3 Task 5: this reconciliation's durable history, in the SAME + // transaction as the authoritative state above. Every step rolls back + // with the rest of the commit, so a Situation's history and its + // authoritative state can never diverge. + if err := applyHistoryCommitTx(ctx, tx, claim.Situation.ID, commit.History, canonicalCommitTime(commit)); err != nil { + return err + } + if err := tx.Commit(); err != nil { return fmt.Errorf("store: commit controller transaction: %w", err) } return nil } +// commitAuthoritativeAttemptTx inserts this commit's new authoritative +// Assessment attempt and its per-Incident coverage tuples, returning the +// attempt id for the current_assessment_id projection (invalid when this +// cycle produced no new attempt at all — a preserve/blocked/still-parked +// cycle, whose current_assessment_id is COALESCEd unchanged). +func commitAuthoritativeAttemptTx(ctx context.Context, tx *sql.Tx, situationID string, commit situation.ControllerCommit) (sql.NullString, error) { + if commit.Attempt.ID == "" { + return sql.NullString{}, nil + } + seq, err := nextAssessmentAttemptSequenceTx(ctx, tx, situationID) + if err != nil { + return sql.NullString{}, err + } + attempt := commit.Attempt + attempt.Sequence = seq + p, err := prepareAssessmentAttempt(attempt, commit.MaterialFactHash) + if err != nil { + return sql.NullString{}, err + } + if err := insertAssessmentAttemptTx(ctx, tx, p); err != nil { + return sql.NullString{}, err + } + for _, cov := range commit.Coverage { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO situation_assessment_coverage (assessment_attempt_id, incident_id, membership_digest, incident_input_digest) + VALUES (?, ?, ?, ?) ON CONFLICT(assessment_attempt_id, incident_id) DO NOTHING`, + attempt.ID, cov.IncidentID, cov.MembershipDigest, cov.IncidentInputDigest); err != nil { + return sql.NullString{}, fmt.Errorf("store: insert assessment coverage: %w", err) + } + } + return sql.NullString{String: attempt.ID, Valid: true}, nil +} + // canonicalCommitTime is the single "now" CommitController's own writes // (Triage decision decided_at passthrough, updated_at) anchor to: the new // attempt's own CompletedAt when one exists (the actual moment this cycle's diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go new file mode 100644 index 0000000..36cd61e --- /dev/null +++ b/internal/store/situation_history.go @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 5: durable Situation history persistence. Everything here +// runs INSIDE Plan 2's existing fenced CommitController transaction — this +// file opens no transaction of its own, so a Situation's authoritative +// state and its durable history can never diverge: either both land or +// neither does. +// ---------------------------------------------------------------------- + +// ErrSituationHistoryConflict means the derived history does not continue +// the Situation's durable history: its first Transition is not the one that +// follows the Situation's current Transition sequence, or its Transitions +// are not contiguous among themselves. The controller derived it against a +// prior Transition that is no longer current, so the whole commit is +// rejected rather than writing a gap into an immutable ledger. +var ErrSituationHistoryConflict = errors.New("store: situation history does not continue the current transition sequence") + +// SupersessionReasonNewerRootProjection is the one supersession reason this +// task writes: a newer committed root projection replaced a still-pending +// older one (R4). It is the store's own closed vocabulary for +// notification_intents.supersession_reason, the same way Plan 2's +// controller owns controller_parked_reason's closed codes. +const SupersessionReasonNewerRootProjection = "newer_root_projection" + +// historyCommitSteps names every durable write step applyHistoryCommitTx +// performs, in order. Tests iterate it to inject a failure after each step +// and prove the whole transaction rolls back (Task 5 brief, Step 1). +var historyCommitSteps = []string{ + "transitions", + "episode_summary", + "transition_stream", + "artifacts", + "intents", + "current_pointer", +} + +// historyCommitFailpoint is a test-only seam: when non-nil it is consulted +// after each step in historyCommitSteps, and a non-nil result aborts the +// enclosing CommitController transaction. It is nil in every production +// build and costs one nil check per step. +var historyCommitFailpoint func(step string) error + +func historyStepDone(step string) error { + if historyCommitFailpoint == nil { + return nil + } + return historyCommitFailpoint(step) +} + +// applyHistoryCommitTx writes one reconciliation's derived history inside +// the caller's already-fenced transaction: every Transition in sequence +// order, one Episode-summary version folded per Transition, one +// transition-stream row per Transition, the consumed operator artifacts' +// journaling cursor (R1), every notification intent (superseding any older +// pending root projection first, R4), and finally the Situation's advanced +// current-Transition pointer. +// +// The Episode summary is re-folded here, from the summary row read inside +// THIS transaction, rather than trusting the single final summary +// HistoryCommit carries: migration 0017's monotonic trigger requires every +// summary write to advance the version by exactly one, so an N-Transition +// commit needs N successive folds, and re-folding from durable truth also +// proves the controller derived its history against the summary that is +// actually current. The final fold is compared against HistoryCommit's own +// canonical summary and a mismatch fails the whole commit closed. +func applyHistoryCommitTx(ctx context.Context, tx *sql.Tx, situationID string, history *situation.HistoryCommit, now time.Time) error { + if history == nil { + return nil + } + if err := validateHistoryCommit(situationID, history); err != nil { + return err + } + + priorSequence, err := currentTransitionSequenceTx(ctx, tx, situationID) + if err != nil { + return err + } + if len(history.Transitions) > 0 && history.Transitions[0].Sequence != priorSequence+1 { + return fmt.Errorf("%w: first transition sequence %d does not follow current sequence %d", + ErrSituationHistoryConflict, history.Transitions[0].Sequence, priorSequence) + } + + for i := range history.Transitions { + if err := insertTransitionTx(ctx, tx, history.Transitions[i]); err != nil { + return err + } + } + if err := historyStepDone("transitions"); err != nil { + return err + } + + if err := foldEpisodeSummaryTx(ctx, tx, situationID, history); err != nil { + return err + } + if err := historyStepDone("episode_summary"); err != nil { + return err + } + + for i := range history.Transitions { + if err := insertTransitionStreamTx(ctx, tx, history.Transitions[i], now); err != nil { + return err + } + } + if err := historyStepDone("transition_stream"); err != nil { + return err + } + + for i := range history.Transitions { + tr := history.Transitions[i] + if tr.OperatorArtifactInputID == nil { + continue + } + if err := journalOperatorArtifactTx(ctx, tx, *tr.OperatorArtifactInputID, tr.ID); err != nil { + return err + } + } + if err := historyStepDone("artifacts"); err != nil { + return err + } + + if err := insertNotificationIntentsTx(ctx, tx, situationID, history.Intents); err != nil { + return err + } + if err := historyStepDone("intents"); err != nil { + return err + } + + if len(history.Transitions) > 0 { + last := history.Transitions[len(history.Transitions)-1] + if _, err := tx.ExecContext(ctx, ` + UPDATE situations SET current_transition_id = ?, current_transition_sequence = ? WHERE id = ?`, + last.ID, last.Sequence, situationID); err != nil { + return fmt.Errorf("store: advance current transition pointer: %w", err) + } + } + return historyStepDone("current_pointer") +} + +// validateHistoryCommit checks the derived history's own coherence before +// any write: every Transition belongs to this Situation, passes its model +// validation, and follows its predecessor by exactly one sequence; the +// summary is present exactly when Transitions are and names the last one; +// and every intent validates and belongs here. +func validateHistoryCommit(situationID string, history *situation.HistoryCommit) error { + for i, tr := range history.Transitions { + if tr.SituationID != situationID { + return fmt.Errorf("store: history transition %d belongs to situation %q, not %q", i, tr.SituationID, situationID) + } + if err := tr.Validate(); err != nil { + return fmt.Errorf("store: history transition %d: %w", i, err) + } + if i > 0 && tr.Sequence != history.Transitions[i-1].Sequence+1 { + return fmt.Errorf("%w: transition %d sequence %d does not follow %d", + ErrSituationHistoryConflict, i, tr.Sequence, history.Transitions[i-1].Sequence) + } + } + switch { + case len(history.Transitions) == 0 && history.Summary != nil: + return errors.New("store: history commit carries a summary with no transitions") + case len(history.Transitions) > 0 && history.Summary == nil: + return errors.New("store: history commit carries transitions with no folded summary") + } + if history.Summary != nil { + if err := history.Summary.Validate(); err != nil { + return fmt.Errorf("store: history episode summary: %w", err) + } + } + for i, intent := range history.Intents { + if err := intent.Validate(); err != nil { + return fmt.Errorf("store: history notification intent %d: %w", i, err) + } + if intent.SituationID == nil || *intent.SituationID != situationID { + return fmt.Errorf("store: history notification intent %d does not belong to situation %q", i, situationID) + } + } + return nil +} + +func currentTransitionSequenceTx(ctx context.Context, tx *sql.Tx, situationID string) (int, error) { + var seq int + err := tx.QueryRowContext(ctx, `SELECT current_transition_sequence FROM situations WHERE id = ?`, situationID).Scan(&seq) + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrNotFound + } + if err != nil { + return 0, fmt.Errorf("store: read current transition sequence: %w", err) + } + return seq, nil +} + +// ---------------------------------------------------------------------- +// Transition writer/scanner +// ---------------------------------------------------------------------- + +func insertTransitionTx(ctx context.Context, tx *sql.Tx, t situationmodel.Transition) error { + contractJSON, err := json.Marshal(t.ActionContract) + if err != nil { + return fmt.Errorf("store: marshal transition action contract: %w", err) + } + journalJSON, err := json.Marshal(t.Journal) + if err != nil { + return fmt.Errorf("store: marshal transition journal: %w", err) + } + projectionJSON, err := json.Marshal(t.Projection) + if err != nil { + return fmt.Errorf("store: marshal transition projection: %w", err) + } + refs := t.EvidenceRefs + if refs == nil { + refs = []string{} + } + refsJSON, err := json.Marshal(refs) + if err != nil { + return fmt.Errorf("store: marshal transition evidence refs: %w", err) + } + var priority any + if t.InterruptionPriority != nil { + priority = string(*t.InterruptionPriority) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO situation_transitions ( + id, situation_id, sequence, input_version, material_fact_hash, assessment_id, + lifecycle, attention, action_contract_json, sufficient_reason_id, interruption_priority, + reason, journal_kind, journal_json, projection_json, operator_artifact_input_id, + evidence_refs_json, actor, drill, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + t.ID, t.SituationID, t.Sequence, t.InputVersion, t.MaterialFactHash, nullableString(t.AssessmentID), + string(t.Lifecycle), string(t.Attention), string(contractJSON), nullableString(t.SufficientReasonID), priority, + string(t.Reason), string(t.JournalKind), string(journalJSON), string(projectionJSON), + nullableString(t.OperatorArtifactInputID), + string(refsJSON), string(t.Actor), boolToInt(t.Drill), canonicalTime(t.CreatedAt)); err != nil { + return fmt.Errorf("store: insert situation transition %s: %w", t.ID, err) + } + return nil +} + +// transitionColumns is the exact SELECT list scanTransition consumes. +const transitionColumns = `id, situation_id, sequence, input_version, material_fact_hash, assessment_id, + lifecycle, attention, action_contract_json, sufficient_reason_id, interruption_priority, + reason, journal_kind, journal_json, projection_json, operator_artifact_input_id, + evidence_refs_json, actor, drill, created_at` + +func scanTransition(row scanner) (situationmodel.Transition, error) { + var t situationmodel.Transition + var assessmentID, reasonID, priority, artifactInputID sql.NullString + var lifecycle, attention, reason, journalKind, actor string + var contractJSON, journalJSON, projectionJSON, refsJSON, createdAt string + var drill int + if err := row.Scan(&t.ID, &t.SituationID, &t.Sequence, &t.InputVersion, &t.MaterialFactHash, &assessmentID, + &lifecycle, &attention, &contractJSON, &reasonID, &priority, + &reason, &journalKind, &journalJSON, &projectionJSON, &artifactInputID, + &refsJSON, &actor, &drill, &createdAt); err != nil { + return situationmodel.Transition{}, err + } + t.Lifecycle = situationmodel.Lifecycle(lifecycle) + t.Attention = situationmodel.Attention(attention) + t.Reason = situationmodel.TransitionReason(reason) + t.JournalKind = situationmodel.JournalKind(journalKind) + t.Actor = situationmodel.TransitionActor(actor) + t.Drill = drill == 1 + if assessmentID.Valid { + t.AssessmentID = &assessmentID.String + } + if reasonID.Valid { + t.SufficientReasonID = &reasonID.String + } + if priority.Valid { + p := situationmodel.InterruptionPriority(priority.String) + t.InterruptionPriority = &p + } + if artifactInputID.Valid { + t.OperatorArtifactInputID = &artifactInputID.String + } + if err := json.Unmarshal([]byte(contractJSON), &t.ActionContract); err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: unmarshal transition %s action contract: %w", t.ID, err) + } + if err := json.Unmarshal([]byte(journalJSON), &t.Journal); err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: unmarshal transition %s journal: %w", t.ID, err) + } + if err := json.Unmarshal([]byte(projectionJSON), &t.Projection); err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: unmarshal transition %s projection: %w", t.ID, err) + } + if err := json.Unmarshal([]byte(refsJSON), &t.EvidenceRefs); err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: unmarshal transition %s evidence refs: %w", t.ID, err) + } + parsed, err := time.Parse(time.RFC3339Nano, createdAt) + if err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: parse transition %s created_at: %w", t.ID, err) + } + t.CreatedAt = parsed.UTC() + return t, nil +} + +// loadCurrentTransitionTx reads the Situation's current Transition (the one +// situations.current_transition_id names), or nil when none exists yet. +func loadCurrentTransitionTx(ctx context.Context, tx *sql.Tx, situationID string) (*situationmodel.Transition, error) { + var id sql.NullString + err := tx.QueryRowContext(ctx, `SELECT current_transition_id FROM situations WHERE id = ?`, situationID).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("store: read current transition id: %w", err) + } + if !id.Valid { + return nil, nil //nolint:nilnil // "no Transition yet" is a legitimate, non-error state. + } + tr, err := scanTransition(tx.QueryRowContext(ctx, + `SELECT `+transitionColumns+` FROM situation_transitions WHERE id = ?`, id.String)) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("store: current transition %s is missing from the ledger", id.String) + } + if err != nil { + return nil, fmt.Errorf("store: read current transition: %w", err) + } + return &tr, nil +} + +// ---------------------------------------------------------------------- +// Episode summary writer/scanner +// ---------------------------------------------------------------------- + +// foldEpisodeSummaryTx folds this commit's Transitions onto the Situation's +// durable current summary, writing one version per Transition, and proves +// the result equals the summary the controller derived. +func foldEpisodeSummaryTx(ctx context.Context, tx *sql.Tx, situationID string, history *situation.HistoryCommit) error { + if len(history.Transitions) == 0 { + return nil + } + prior, err := loadEpisodeSummaryTx(ctx, tx, situationID) + if err != nil { + return err + } + exists := prior != nil + for i, tr := range history.Transitions { + folded, err := situation.ProjectEpisode(prior, tr) + if err != nil { + return fmt.Errorf("store: fold episode summary at transition %d: %w", i, err) + } + if err := writeEpisodeSummaryTx(ctx, tx, folded, exists); err != nil { + return err + } + exists = true + prior = &folded + } + + got, err := json.Marshal(prior) + if err != nil { + return fmt.Errorf("store: marshal folded episode summary: %w", err) + } + want, err := json.Marshal(history.Summary) + if err != nil { + return fmt.Errorf("store: marshal derived episode summary: %w", err) + } + if string(got) != string(want) { + return fmt.Errorf("store: folded episode summary %s does not match the committed summary %s", got, want) + } + return nil +} + +func writeEpisodeSummaryTx(ctx context.Context, tx *sql.Tx, s situationmodel.EpisodeSummary, exists bool) error { + payload, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("store: marshal episode summary: %w", err) + } + query := `INSERT INTO situation_episode_summaries (situation_id, version, source_transition_sequence, summary_json, updated_at) + VALUES (?, ?, ?, ?, ?)` + if exists { + query = `UPDATE situation_episode_summaries + SET version = ?, source_transition_sequence = ?, summary_json = ?, updated_at = ? + WHERE situation_id = ?` + if _, err := tx.ExecContext(ctx, query, s.Version, s.SourceTransitionSequence, + string(payload), canonicalTime(s.UpdatedAt), s.SituationID); err != nil { + return fmt.Errorf("store: update episode summary for %s: %w", s.SituationID, err) + } + return nil + } + if _, err := tx.ExecContext(ctx, query, s.SituationID, s.Version, s.SourceTransitionSequence, + string(payload), canonicalTime(s.UpdatedAt)); err != nil { + return fmt.Errorf("store: insert episode summary for %s: %w", s.SituationID, err) + } + return nil +} + +func loadEpisodeSummaryTx(ctx context.Context, tx *sql.Tx, situationID string) (*situationmodel.EpisodeSummary, error) { + var payload string + err := tx.QueryRowContext(ctx, + `SELECT summary_json FROM situation_episode_summaries WHERE situation_id = ?`, situationID).Scan(&payload) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil //nolint:nilnil // "no summary yet" is a legitimate, non-error state. + } + if err != nil { + return nil, fmt.Errorf("store: read episode summary: %w", err) + } + var summary situationmodel.EpisodeSummary + if err := json.Unmarshal([]byte(payload), &summary); err != nil { + return nil, fmt.Errorf("store: unmarshal episode summary for %s: %w", situationID, err) + } + return &summary, nil +} + +// ---------------------------------------------------------------------- +// Transition stream writer +// ---------------------------------------------------------------------- + +// transitionStreamID derives the stdout-stream outbox row's identity from +// the Transition it records, so the row is as deterministic as the +// Transition itself. +func transitionStreamID(transitionID string) string { return "stream-" + transitionID } + +func insertTransitionStreamTx(ctx context.Context, tx *sql.Tx, t situationmodel.Transition, now time.Time) error { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO situation_transition_stream (id, transition_id, situation_id, sequence, status, created_at) + VALUES (?, ?, ?, ?, 'pending', ?)`, + transitionStreamID(t.ID), t.ID, t.SituationID, t.Sequence, canonicalTime(now)); err != nil { + return fmt.Errorf("store: insert transition stream row for %s: %w", t.ID, err) + } + return nil +} + +// ---------------------------------------------------------------------- +// Operator-artifact journaling cursor (R1) +// ---------------------------------------------------------------------- + +// journalOperatorArtifactTx marks one applied, pending artifact input as +// journaled by the Transition that consumed it. A row that is not pending +// any more (another commit consumed it, or R2 recorded it against a +// terminal owner) fails the whole commit closed rather than silently +// journaling nothing. +func journalOperatorArtifactTx(ctx context.Context, tx *sql.Tx, inputID, transitionID string) error { + res, err := tx.ExecContext(ctx, ` + UPDATE situation_input_outbox + SET journal_state = 'journaled', journaled_transition_id = ? + WHERE id = ? AND journal_state = 'pending'`, transitionID, inputID) + if err != nil { + return fmt.Errorf("store: journal operator artifact %s: %w", inputID, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("store: count journaled operator artifact %s: %w", inputID, err) + } + if n != 1 { + return fmt.Errorf("store: operator artifact %s is no longer pending journaling", inputID) + } + return nil +} + +// ---------------------------------------------------------------------- +// Notification intents +// ---------------------------------------------------------------------- + +// insertNotificationIntentsTx supersedes any still-pending root projection +// this commit replaces and then inserts every planned intent. Migration +// 0018 allows at most one pending, unsuperseded root_sync per Situation, so +// the supersession and its replacement have to happen in this one +// transaction or not at all (R4). +func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID string, intents []situationmodel.NotificationIntent) error { + if len(intents) == 0 { + return nil + } + for _, intent := range intents { + if intent.EffectClass != situationmodel.EffectRootSync || intent.Status != situationmodel.IntentPending { + continue + } + if err := supersedePendingRootSyncTx(ctx, tx, situationID, intent.ID); err != nil { + return err + } + break // PlanNotificationIntents never plans two root projections per commit. + } + for i := range intents { + if err := insertNotificationIntentTx(ctx, tx, intents[i]); err != nil { + return err + } + } + return nil +} + +// supersedePendingRootSyncTx retires every currently-pending root_sync for +// situationID in favour of replacementID. replacementID is inserted later +// in this same transaction, so foreign-key enforcement is deferred to +// COMMIT for the duration: the self-referencing replacement_intent_id FK +// and the "at most one pending root_sync" index would otherwise make the +// two writes impossible to order. Deferral changes when a violation is +// reported, never whether the transaction is atomic. +func supersedePendingRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, replacementID string) error { + var pending int + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, situationID).Scan(&pending); err != nil { + return fmt.Errorf("store: count pending root projections: %w", err) + } + if pending == 0 { + return nil + } + if _, err := tx.ExecContext(ctx, `PRAGMA defer_foreign_keys = ON`); err != nil { + return fmt.Errorf("store: defer foreign keys for root supersession: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE notification_intents + SET status = 'superseded', supersession_reason = ?, replacement_intent_id = ?, + claim_owner = NULL, lease_expires_at = NULL, retry_at = NULL + WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, + SupersessionReasonNewerRootProjection, replacementID, situationID); err != nil { + return fmt.Errorf("store: supersede pending root projections: %w", err) + } + return nil +} + +func insertNotificationIntentTx(ctx context.Context, tx *sql.Tx, n situationmodel.NotificationIntent) error { + var priority any + if n.InterruptionPriority != nil { + priority = string(*n.InterruptionPriority) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + summary_version, gap_generation, requires_root, main_channel_poke, interruption_priority, + contract_deadline_at, client_message_id, status, claim_owner, claim_token, lease_expires_at, + attempt_count, last_error_class, retry_at, supersession_reason, replacement_intent_id, + delivered_as, channel, message_ts, created_at, delivered_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + n.ID, n.IdempotencyKey, string(n.EffectClass), nullableString(n.SituationID), nullableString(n.TransitionID), + nullableInt(n.TransitionSequence), nullableInt(n.SummaryVersion), nullableString(n.GapGeneration), + boolToInt(n.RequiresRoot), boolToInt(n.MainChannelPoke), priority, + nullableTimePtr(n.ContractDeadlineAt), n.ClientMessageID, string(n.Status), + nullableString(n.ClaimOwner), n.ClaimToken, nullableTimePtr(n.LeaseExpiresAt), + n.AttemptCount, nullableString(n.LastErrorClass), nullableTimePtr(n.RetryAt), + nullableString(n.SupersessionReason), nullableString(n.ReplacementIntentID), + nullableString(n.DeliveredAs), nullableString(n.Channel), nullableString(n.MessageTS), + canonicalTime(n.CreatedAt), nullableTimePtr(n.DeliveredAt)); err != nil { + return fmt.Errorf("store: insert notification intent %s: %w", n.ID, err) + } + return nil +} + +// notificationIntentColumns is the exact SELECT list scanNotificationIntent +// consumes. +const notificationIntentColumns = `id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + summary_version, gap_generation, requires_root, main_channel_poke, interruption_priority, + contract_deadline_at, client_message_id, status, claim_owner, claim_token, lease_expires_at, + attempt_count, last_error_class, retry_at, supersession_reason, replacement_intent_id, + delivered_as, channel, message_ts, created_at, delivered_at` + +func scanNotificationIntent(row scanner) (situationmodel.NotificationIntent, error) { + var n situationmodel.NotificationIntent + var effectClass, status, clientMessageID, createdAt string + var situationID, transitionID, gapGeneration, priority, claimOwner sql.NullString + var lastErrorClass, supersessionReason, replacementID, deliveredAs, channel, messageTS sql.NullString + var contractDeadlineAt, leaseExpiresAt, retryAt, deliveredAt sql.NullString + var transitionSequence, summaryVersion sql.NullInt64 + var requiresRoot, mainChannelPoke int + if err := row.Scan(&n.ID, &n.IdempotencyKey, &effectClass, &situationID, &transitionID, &transitionSequence, + &summaryVersion, &gapGeneration, &requiresRoot, &mainChannelPoke, &priority, + &contractDeadlineAt, &clientMessageID, &status, &claimOwner, &n.ClaimToken, &leaseExpiresAt, + &n.AttemptCount, &lastErrorClass, &retryAt, &supersessionReason, &replacementID, + &deliveredAs, &channel, &messageTS, &createdAt, &deliveredAt); err != nil { + return situationmodel.NotificationIntent{}, err + } + n.EffectClass = situationmodel.EffectClass(effectClass) + n.Status = situationmodel.IntentStatus(status) + n.ClientMessageID = clientMessageID + n.RequiresRoot = requiresRoot == 1 + n.MainChannelPoke = mainChannelPoke == 1 + for _, f := range []struct { + src sql.NullString + dst **string + }{ + {situationID, &n.SituationID}, {transitionID, &n.TransitionID}, {gapGeneration, &n.GapGeneration}, + {claimOwner, &n.ClaimOwner}, {lastErrorClass, &n.LastErrorClass}, {supersessionReason, &n.SupersessionReason}, + {replacementID, &n.ReplacementIntentID}, {deliveredAs, &n.DeliveredAs}, {channel, &n.Channel}, + {messageTS, &n.MessageTS}, + } { + if f.src.Valid { + v := f.src.String + *f.dst = &v + } + } + if transitionSequence.Valid { + v := int(transitionSequence.Int64) + n.TransitionSequence = &v + } + if summaryVersion.Valid { + v := int(summaryVersion.Int64) + n.SummaryVersion = &v + } + if priority.Valid { + p := situationmodel.InterruptionPriority(priority.String) + n.InterruptionPriority = &p + } + for _, f := range []struct { + src sql.NullString + dst **time.Time + }{ + {contractDeadlineAt, &n.ContractDeadlineAt}, {leaseExpiresAt, &n.LeaseExpiresAt}, + {retryAt, &n.RetryAt}, {deliveredAt, &n.DeliveredAt}, + } { + parsed, err := timePtr(f.src) + if err != nil { + return situationmodel.NotificationIntent{}, err + } + *f.dst = parsed + } + parsed, err := time.Parse(time.RFC3339Nano, createdAt) + if err != nil { + return situationmodel.NotificationIntent{}, fmt.Errorf("store: parse notification intent %s created_at: %w", n.ID, err) + } + n.CreatedAt = parsed.UTC() + return n, nil +} + +// ---------------------------------------------------------------------- +// Small local helpers. +// ---------------------------------------------------------------------- + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + +func nullableInt(v *int) any { + if v == nil { + return nil + } + return *v +} + +// prefixedTransitionColumns is transitionColumns qualified with the `t` +// table alias, for the joined reads that also select another table's +// columns. Derived from transitionColumns itself so the two can never +// drift apart. +var prefixedTransitionColumns = qualifyColumns(transitionColumns, "t") + +func qualifyColumns(list, alias string) string { + parts := strings.Split(list, ",") + for i := range parts { + parts[i] = alias + "." + strings.TrimSpace(parts[i]) + } + return strings.Join(parts, ", ") +} + +// scanStreamEntry scans one `SELECT st.id, ` row: +// the stream row's own id into streamID, then the joined Transition. +func scanStreamEntry(rows *sql.Rows, streamID *string) (situationmodel.Transition, error) { + tr, err := scanTransition(prefixedScanner{rows: rows, prefix: []any{streamID}}) + if err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: scan pending transition stream entry: %w", err) + } + return tr, nil +} + +// prefixedScanner lets scanTransition consume a row that carries extra +// leading columns (store.scanner's own shape), without duplicating its +// column list. +type prefixedScanner struct { + rows *sql.Rows + prefix []any +} + +func (p prefixedScanner) Scan(dest ...any) error { + all := make([]any, 0, len(p.prefix)+len(dest)) + all = append(all, p.prefix...) + all = append(all, dest...) + return p.rows.Scan(all...) +} diff --git a/internal/store/situation_history_test.go b/internal/store/situation_history_test.go new file mode 100644 index 0000000..2febbfe --- /dev/null +++ b/internal/store/situation_history_test.go @@ -0,0 +1,993 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "sync" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 5: fixtures for the fenced history commit. Every helper is +// prefixed `sh` (situation history) so it never collides with the store +// package's existing Plan 1/2 test helpers. +// ---------------------------------------------------------------------- + +// shRunningTriageContract is a valid nonterminal Operator contract in which +// AlertINT is currently running Acute Triage. +func shRunningTriageContract(next time.Time) situationmodel.ActionContract { + action := situationmodel.AlertINTActionRunAcuteTriage + status := situationmodel.AlertINTStatusRunning + return situationmodel.ActionContract{ + NextActor: situationmodel.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: timePtrValue(next), + NextUpdateOn: []situationmodel.NextUpdateOn{situationmodel.NextUpdateOnTriageOutcome}, + } +} + +// shOperatorContract hands the next move to a human while AlertINT's own +// Acute Triage work continues — a material Operator-contract change against +// shRunningTriageContract. +func shOperatorContract(next time.Time) situationmodel.ActionContract { + op := situationmodel.OperatorActionInvestigateSituation + action := situationmodel.AlertINTActionRunAcuteTriage + status := situationmodel.AlertINTStatusRunning + return situationmodel.ActionContract{ + NextActor: situationmodel.NextActorOperator, + AlertINTAction: &action, + AlertINTStatus: &status, + OperatorActionRequired: &op, + NextUpdateAt: timePtrValue(next), + NextUpdateOn: []situationmodel.NextUpdateOn{situationmodel.NextUpdateOnMaterialInput}, + } +} + +func shConclusion() situationmodel.AssessmentConclusion { + return situationmodel.AssessmentConclusion{ + Persistence: situationmodel.PersistenceSustained, + Impact: situationmodel.ImpactSuspected, + Novelty: situationmodel.NoveltyNew, + Causality: situationmodel.CausalityCorrelated, + EvidenceQuality: situationmodel.EvidenceQualityComplete, + LimitationCodes: []string{}, + SufficientReasonCode: "critical_anchor", + SufficientReasonSummary: "Confirmed active critical source severity.", + } +} + +func shAssessment(contract situationmodel.ActionContract, concl situationmodel.AssessmentConclusion, + lifecycle situationmodel.Lifecycle, attention situationmodel.Attention) situationmodel.Assessment { + a := situationmodel.Assessment{ + SchemaVersion: situationmodel.AssessmentSchemaVersion, + Persistence: concl.Persistence, + Impact: concl.Impact, + Novelty: concl.Novelty, + Causality: concl.Causality, + Attention: attention, + Lifecycle: lifecycle, + EvidenceQuality: concl.EvidenceQuality, + ActionContract: contract, + Limitations: []situationmodel.Limitation{}, + Cadence: situationmodel.CadenceFast, + } + if lifecycle.Terminal() { + a.Cadence = situationmodel.Cadence("") + } + if concl.SufficientReasonCode != "" { + a.SufficientReason = &situationmodel.SufficientReason{ + Code: concl.SufficientReasonCode, + CandidateID: "candidate-" + concl.SufficientReasonCode, + Summary: concl.SufficientReasonSummary, + EvidenceRefs: []string{"fact-anchor"}, + } + } + return a +} + +// shCycle is one prepared reconciliation: the ControllerCommit Plan 2 would +// hand CommitController, plus the derived history Plan 3 attaches to it. +type shCycle struct { + Commit situation.ControllerCommit + Change situation.AuthoritativeChange + Publish situation.PublicationInput +} + +// shPrepare builds one realistic reconciliation for claim: a fresh +// authoritative attempt, the matching Assessment/contract, and the history +// BuildHistoryCommit derives from them. +func shPrepare(t *testing.T, claim situation.Claim, contract situationmodel.ActionContract, + lifecycle situationmodel.Lifecycle, attention situationmodel.Attention, now time.Time) shCycle { + t.Helper() + sit := claim.Situation + sit.Lifecycle = lifecycle + sit.Attention = attention + + concl := shConclusion() + assessment := shAssessment(contract, concl, lifecycle, attention) + commit := basicControllerCommit(sit.ID, sit.InputVersion, now) + commit.Assessment = assessment + commit.Attempt.Validated = mustMarshalJSON(t, assessment) + commit.Lifecycle = lifecycle + commit.Attention = attention + commit.MaterialFactHash = "sha256:material-" + sit.ID + commit.ConsumedDueReasons = claim.Situation.DueReasons + + projection := situationmodel.ProjectionFacts{ + EffectiveStartedAt: sit.EffectiveStartedAt, + EffectiveStartedAtBasis: sit.EffectiveStartedAtBasis, + Assessment: &concl, + } + if lifecycle.Terminal() { + projection.TerminalAt = timePtrValue(now) + sit.TerminalAt = timePtrValue(now) + commit.TerminalAt = timePtrValue(now) + if lifecycle == situationmodel.LifecycleClosedUnknown { + reason := situationmodel.TerminalReasonObservationDeadline + projection.TerminalReason = &reason + sit.TerminalReason = &reason + commit.TerminalReason = &reason + } + } + + change := situation.AuthoritativeChange{ + Situation: sit, + AssessmentID: &commit.Attempt.ID, + Assessment: assessment, + Derivation: situationmodel.DerivationDeterministic, + Projection: projection, + MaterialFactHash: commit.MaterialFactHash, + EvidenceRefs: []string{"fact-a"}, + Now: now, + } + publish := situation.PublicationInput{ + Situation: sit, + SlackFloor: situationmodel.InterruptionLow, + RepageCooldown: 15 * time.Minute, + Now: now, + } + if !lifecycle.Terminal() { + publish.ContractDeadlineAt = contract.NextUpdateAt + } + return shCycle{Commit: commit, Change: change, Publish: publish} +} + +// shDerive runs Task 4's composition and attaches the result to the commit. +func shDerive(t *testing.T, c shCycle) situation.ControllerCommit { + t.Helper() + history, err := situation.BuildHistoryCommit(c.Change, c.Publish) + if err != nil { + t.Fatalf("BuildHistoryCommit: %v", err) + } + commit := c.Commit + commit.History = &history + return commit +} + +// shSeedPendingArtifact inserts one applied, journal_state='pending' +// situation_input_outbox row owned by situationID, and returns the +// OperatorArtifactInput the controller would hand BuildTransitions for it. +func shSeedPendingArtifact(t *testing.T, st *Store, situationID, incidentID, groupKey, inputID, kind string, + appliedInputVersion int, occurredAt time.Time) situation.OperatorArtifactInput { + t.Helper() + ctx := context.Background() + artifact := situation.OperatorArtifactInput{ + InputID: inputID, + Kind: kind, + AppliedInputVersion: appliedInputVersion, + OccurredAt: occurredAt.UTC(), + AttributedActor: "operator@example.com", + Headline: "Operator note recorded", + Detail: "Rollback started.", + } + var annotationID, verdictID any + switch kind { + case "operator_annotation_recorded": + id, err := insertAnnotationRow(ctx, st, incidentID) + if err != nil { + t.Fatalf("insert annotation row: %v", err) + } + annotationID = id + artifact.AnnotationID = shStringPtr(strconv.FormatInt(id, 10)) + case "captured_verdict_recorded": + id, err := insertVerdictRow(ctx, st, incidentID, appliedInputVersion) + if err != nil { + t.Fatalf("insert verdict row: %v", err) + } + verdictID = id + artifact.VerdictID = shStringPtr(strconv.FormatInt(id, 10)) + default: + t.Fatalf("unsupported artifact kind %q", kind) + } + if _, err := st.db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, status, + applied_situation_id, applied_at, applied_input_version, + annotation_id, verdict_id, journal_state + ) VALUES (?, ?, ?, ?, ?, ?, 'applied', ?, ?, ?, ?, ?, 'pending')`, + inputID, "idem:"+inputID, incidentID, kind, groupKey, canonicalTime(occurredAt), + situationID, canonicalTime(occurredAt), appliedInputVersion, annotationID, verdictID); err != nil { + t.Fatalf("seed pending artifact %s: %v", inputID, err) + } + return artifact +} + +func shStringPtr(s string) *string { return &s } + +// ---------------------------------------------------------------------- +// Assertion helpers reading the durable history back. +// ---------------------------------------------------------------------- + +func shTransitionSequences(t *testing.T, st *Store, situationID string) []int { + t.Helper() + rows, err := st.db.QueryContext(context.Background(), + `SELECT sequence FROM situation_transitions WHERE situation_id = ? ORDER BY sequence`, situationID) + if err != nil { + t.Fatalf("read transition sequences: %v", err) + } + defer func() { _ = rows.Close() }() + out := []int{} + for rows.Next() { + var seq int + if err := rows.Scan(&seq); err != nil { + t.Fatalf("scan transition sequence: %v", err) + } + out = append(out, seq) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate transition sequences: %v", err) + } + return out +} + +func shCountRows(t *testing.T, st *Store, query string, args ...any) int { + t.Helper() + var n int + if err := st.db.QueryRowContext(context.Background(), query, args...).Scan(&n); err != nil { + t.Fatalf("count query %q: %v", query, err) + } + return n +} + +func shCurrentPointer(t *testing.T, st *Store, situationID string) (string, int) { + t.Helper() + var id nullStringForTest + var seq int + if err := st.db.QueryRowContext(context.Background(), + `SELECT current_transition_id, current_transition_sequence FROM situations WHERE id = ?`, + situationID).Scan(&id, &seq); err != nil { + t.Fatalf("read current transition pointer: %v", err) + } + return id.String, seq +} + +// nullStringForTest keeps the pointer read above readable without pulling +// database/sql into every assertion. +type nullStringForTest struct { + String string + Valid bool +} + +func (n *nullStringForTest) Scan(v any) error { + if v == nil { + n.String, n.Valid = "", false + return nil + } + switch s := v.(type) { + case string: + n.String, n.Valid = s, true + case []byte: + n.String, n.Valid = string(s), true + default: + return fmt.Errorf("unexpected type %T", v) + } + return nil +} + +// ---------------------------------------------------------------------- +// Step 1: atomicity. +// ---------------------------------------------------------------------- + +// TestControllerCommitHistoryPersistsEveryDurableRecordInOneTransaction is +// the whole-commit success case: one transaction updates Plan 2's +// projection AND inserts every Transition in contiguous sequence order, one +// Episode-summary version per Transition, one transition-stream row per +// Transition, every planned intent, and the advanced current pointer. +func TestControllerCommitHistoryPersistsEveryDurableRecordInOneTransaction(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "group-history-ok", now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + commit := shDerive(t, cycle) + if len(commit.History.Transitions) != 1 { + t.Fatalf("derived transitions = %d, want 1", len(commit.History.Transitions)) + } + if err := st.CommitController(ctx, claim, commit); err != nil { + t.Fatalf("CommitController: %v", err) + } + + tr := commit.History.Transitions[0] + if got := shTransitionSequences(t, st, sitID); len(got) != 1 || got[0] != 1 { + t.Fatalf("transition sequences = %v, want [1]", got) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transition_stream WHERE situation_id = ?`, sitID); n != 1 { + t.Fatalf("transition stream rows = %d, want 1", n) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`, sitID); n != len(commit.History.Intents) { + t.Fatalf("notification intents = %d, want %d", n, len(commit.History.Intents)) + } + var version, sourceSeq int + var summaryJSON string + if err := st.db.QueryRowContext(ctx, + `SELECT version, source_transition_sequence, summary_json FROM situation_episode_summaries WHERE situation_id = ?`, + sitID).Scan(&version, &sourceSeq, &summaryJSON); err != nil { + t.Fatalf("read episode summary: %v", err) + } + if version != 1 || sourceSeq != tr.Sequence { + t.Fatalf("episode summary (version, source_sequence) = (%d,%d), want (1,%d)", version, sourceSeq, tr.Sequence) + } + wantSummary, err := json.Marshal(commit.History.Summary) + if err != nil { + t.Fatalf("marshal expected summary: %v", err) + } + if summaryJSON != string(wantSummary) { + t.Fatalf("persisted summary_json = %s, want %s", summaryJSON, wantSummary) + } + if id, seq := shCurrentPointer(t, st, sitID); id != tr.ID || seq != tr.Sequence { + t.Fatalf("current transition pointer = (%q,%d), want (%q,%d)", id, seq, tr.ID, tr.Sequence) + } + // The immutable journal entry for this Transition landed too, and knows + // it may not be delivered before the root exists. + thread := shIntentOfClass(t, commit.History.Intents, "thread_append") + storedThread, err := st.GetNotificationIntent(ctx, thread.ID) + if err != nil { + t.Fatalf("GetNotificationIntent(thread_append): %v", err) + } + if !storedThread.RequiresRoot || storedThread.MainChannelPoke { + t.Fatalf("thread_append intent = %+v, want requires_root and no poke", storedThread) + } + if storedThread.TransitionSequence == nil || *storedThread.TransitionSequence != tr.Sequence { + t.Fatalf("thread_append transition sequence = %v, want %d", storedThread.TransitionSequence, tr.Sequence) + } + // Plan 2's own projection still landed in the same transaction. + var currentAssessmentID string + if err := st.db.QueryRowContext(ctx, `SELECT current_assessment_id FROM situations WHERE id = ?`, sitID). + Scan(¤tAssessmentID); err != nil { + t.Fatalf("read current assessment id: %v", err) + } + if currentAssessmentID != commit.Attempt.ID { + t.Fatalf("current_assessment_id = %q, want %q", currentAssessmentID, commit.Attempt.ID) + } +} + +// TestControllerCommitHistoryJournalsEveryPendingArtifactInOneCommit pins +// R1: two artifacts applied between two controller cycles are journaled in +// one commit, in (applied_input_version, occurred_at, id) order, before the +// controller-state Transition, each carrying its own Transition ID. +func TestControllerCommitHistoryJournalsEveryPendingArtifactInOneCommit(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + group := "group-history-artifacts" + sitID := newSituationForGroup(t, st, group, now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + first := shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-art-1", + "operator_annotation_recorded", claim.Situation.InputVersion, now.Add(-2*time.Minute)) + second := shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-art-2", + "captured_verdict_recorded", claim.Situation.InputVersion, now.Add(-time.Minute)) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + cycle.Change.OperatorArtifacts = []situation.OperatorArtifactInput{first, second} + commit := shDerive(t, cycle) + if len(commit.History.Transitions) != 3 { + t.Fatalf("derived transitions = %d, want 3 (two artifacts then the controller state)", len(commit.History.Transitions)) + } + if err := st.CommitController(ctx, claim, commit); err != nil { + t.Fatalf("CommitController: %v", err) + } + + if got := shTransitionSequences(t, st, sitID); len(got) != 3 || got[0] != 1 || got[1] != 2 || got[2] != 3 { + t.Fatalf("transition sequences = %v, want [1 2 3]", got) + } + for i, artifact := range []situation.OperatorArtifactInput{first, second} { + var state string + var journaledTransitionID string + if err := st.db.QueryRowContext(ctx, + `SELECT journal_state, journaled_transition_id FROM situation_input_outbox WHERE id = ?`, + artifact.InputID).Scan(&state, &journaledTransitionID); err != nil { + t.Fatalf("read artifact %s: %v", artifact.InputID, err) + } + if state != "journaled" { + t.Fatalf("artifact %s journal_state = %q, want journaled", artifact.InputID, state) + } + if want := commit.History.Transitions[i].ID; journaledTransitionID != want { + t.Fatalf("artifact %s journaled_transition_id = %q, want %q", artifact.InputID, journaledTransitionID, want) + } + } + // One Episode-summary version per Transition: three folds from nothing. + var version int + if err := st.db.QueryRowContext(ctx, + `SELECT version FROM situation_episode_summaries WHERE situation_id = ?`, sitID).Scan(&version); err != nil { + t.Fatalf("read episode summary version: %v", err) + } + if version != 3 { + t.Fatalf("episode summary version = %d, want 3 (one fold per Transition)", version) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transition_stream WHERE situation_id = ?`, sitID); n != 3 { + t.Fatalf("transition stream rows = %d, want 3", n) + } +} + +// TestControllerCommitHistoryFailedWriteLeavesNoPartialState injects a +// failure after each durable history write step and proves the whole +// transaction rolls back: no orphan Transition, no orphan summary, stream +// row, or intent, no journaled artifact, and no advanced current pointer. +func TestControllerCommitHistoryFailedWriteLeavesNoPartialState(t *testing.T) { + for _, step := range historyCommitSteps { + t.Run(step, func(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + group := "group-history-fail-" + step + sitID := newSituationForGroup(t, st, group, now) + claim := claimSituation(t, st, sitID, "controller-a", now) + artifact := shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-fail-"+step, + "operator_annotation_recorded", claim.Situation.InputVersion, now.Add(-time.Minute)) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + cycle.Change.OperatorArtifacts = []situation.OperatorArtifactInput{artifact} + commit := shDerive(t, cycle) + + injected := errors.New("injected failure at " + step) + historyCommitFailpoint = func(at string) error { + if at == step { + return injected + } + return nil + } + t.Cleanup(func() { historyCommitFailpoint = nil }) + + if err := st.CommitController(ctx, claim, commit); !errors.Is(err, injected) { + t.Fatalf("CommitController with a failure at %q = %v, want the injected error", step, err) + } + historyCommitFailpoint = nil + + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("transitions after rollback = %d, want 0", n) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_episode_summaries WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("episode summaries after rollback = %d, want 0", n) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transition_stream WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("stream rows after rollback = %d, want 0", n) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("intents after rollback = %d, want 0", n) + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM situation_input_outbox WHERE id = ? AND journal_state = 'pending'`, + artifact.InputID); n != 1 { + t.Fatalf("artifact journal_state after rollback: pending rows = %d, want 1", n) + } + if id, seq := shCurrentPointer(t, st, sitID); id != "" || seq != 0 { + t.Fatalf("current transition pointer after rollback = (%q,%d), want (\"\",0)", id, seq) + } + // Plan 2's own projection must have rolled back too. + var currentAssessmentID nullStringForTest + if err := st.db.QueryRowContext(ctx, `SELECT current_assessment_id FROM situations WHERE id = ?`, sitID). + Scan(¤tAssessmentID); err != nil { + t.Fatalf("read current assessment id: %v", err) + } + if currentAssessmentID.Valid { + t.Fatalf("current_assessment_id after rollback = %q, want NULL", currentAssessmentID.String) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_assessment_attempts WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("assessment attempts after rollback = %d, want 0", n) + } + }) + } +} + +// ---------------------------------------------------------------------- +// Step 2: fencing and concurrency. +// ---------------------------------------------------------------------- + +// TestControllerCommitFenceRejectsHistoryOnStaleClaim proves a stale lease +// owner, claim token, or input version rejects the WHOLE commit — history +// included — never just Plan 2's own fields. +func TestControllerCommitFenceRejectsHistoryOnStaleClaim(t *testing.T) { + cases := []struct { + name string + mutate func(*situation.Claim) + wantErr error + }{ + {"owner", func(c *situation.Claim) { c.ClaimOwner = "someone-else" }, situationmodel.ErrSituationLeaseLost}, + {"token", func(c *situation.Claim) { c.ClaimToken++ }, situationmodel.ErrSituationLeaseLost}, + {"input_version", func(c *situation.Claim) { c.Situation.InputVersion++ }, ErrSituationVersionConflict}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "group-history-fence-"+tc.name, now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + commit := shDerive(t, cycle) + + stale := claim + tc.mutate(&stale) + if err := st.CommitController(ctx, stale, commit); !errors.Is(err, tc.wantErr) { + t.Fatalf("CommitController with a stale %s = %v, want %v", tc.name, err, tc.wantErr) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("transitions after a fenced rejection = %d, want 0", n) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("intents after a fenced rejection = %d, want 0", n) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_episode_summaries WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("episode summaries after a fenced rejection = %d, want 0", n) + } + }) + } +} + +// TestControllerCommitConcurrentRacePicksOneContiguousWinner races two valid +// commits for one Situation: exactly one lands, sequences stay contiguous, +// and no Transition or intent is duplicated. +func TestControllerCommitConcurrentRacePicksOneContiguousWinner(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "group-history-race", now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + commitA := shDerive(t, cycle) + commitB := shDerive(t, cycle) + commitB.Attempt.ID = commitA.Attempt.ID // same derivation, same cycle, same claim. + + var wg sync.WaitGroup + errs := make([]error, 2) + for i, c := range []situation.ControllerCommit{commitA, commitB} { + wg.Add(1) + go func(idx int, commit situation.ControllerCommit) { + defer wg.Done() + errs[idx] = st.CommitController(ctx, claim, commit) + }(i, c) + } + wg.Wait() + + won := 0 + for _, err := range errs { + switch { + case err == nil: + won++ + case errors.Is(err, situationmodel.ErrSituationLeaseLost): + default: + t.Fatalf("racing commit failed with an unexpected error: %v", err) + } + } + if won != 1 { + t.Fatalf("winning commits = %d, want exactly 1 (errors: %v)", won, errs) + } + if got := shTransitionSequences(t, st, sitID); len(got) != 1 || got[0] != 1 { + t.Fatalf("transition sequences after the race = %v, want [1]", got) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`, sitID); n != len(commitA.History.Intents) { + t.Fatalf("intents after the race = %d, want %d", n, len(commitA.History.Intents)) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transition_stream WHERE situation_id = ?`, sitID); n != 1 { + t.Fatalf("stream rows after the race = %d, want 1", n) + } +} + +// TestControllerCommitHistoryVerbatimReplayFailsClosed pins Plan 2's replay +// boundary 9 for history: replaying a landed commit with the same (now +// stale) claim fails closed and changes nothing. +func TestControllerCommitHistoryVerbatimReplayFailsClosed(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "group-history-replay", now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + commit := shDerive(t, cycle) + if err := st.CommitController(ctx, claim, commit); err != nil { + t.Fatalf("first CommitController: %v", err) + } + before := shTransitionSequences(t, st, sitID) + intentsBefore := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`, sitID) + + if err := st.CommitController(ctx, claim, commit); !errors.Is(err, situationmodel.ErrSituationLeaseLost) { + t.Fatalf("verbatim replay = %v, want ErrSituationLeaseLost", err) + } + after := shTransitionSequences(t, st, sitID) + if len(after) != len(before) { + t.Fatalf("transitions after replay = %v, want unchanged %v", after, before) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`, sitID); n != intentsBefore { + t.Fatalf("intents after replay = %d, want unchanged %d", n, intentsBefore) + } +} + +// TestControllerCommitHistorySupersedesPendingRootSync proves a second +// commit's root projection atomically supersedes the first commit's still +// pending one — migration 0018's "at most one pending, unsuperseded +// root_sync per Situation" index would otherwise reject the whole commit. +func TestControllerCommitHistorySupersedesPendingRootSync(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + group := "group-history-supersede" + sitID := newSituationForGroup(t, st, group, now) + + claim := claimSituation(t, st, sitID, "controller-a", now) + first := shDerive(t, shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now)) + if err := st.CommitController(ctx, claim, first); err != nil { + t.Fatalf("first CommitController: %v", err) + } + firstRoot := shIntentOfClass(t, first.History.Intents, "root_sync") + + // Second cycle: same input version, a materially changed contract. + later := now.Add(time.Minute) + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim2 := claimSituation(t, st, sitID, "controller-a", later) + cycle2 := shPrepare(t, claim2, shOperatorContract(later.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, later) + cycle2.Change.PriorTransition = &first.History.Transitions[0] + cycle2.Change.PriorSummary = first.History.Summary + cycle2.Publish.PriorTransition = &first.History.Transitions[0] + cycle2.Publish.RootPublished = true + commit2 := shDerive(t, cycle2) + if err := st.CommitController(ctx, claim2, commit2); err != nil { + t.Fatalf("second CommitController: %v", err) + } + + var status, supersessionReason, replacement string + if err := st.db.QueryRowContext(ctx, + `SELECT status, COALESCE(supersession_reason,''), COALESCE(replacement_intent_id,'') FROM notification_intents WHERE id = ?`, + firstRoot.ID).Scan(&status, &supersessionReason, &replacement); err != nil { + t.Fatalf("read first root intent: %v", err) + } + if status != "superseded" { + t.Fatalf("first root_sync status = %q, want superseded", status) + } + secondRoot := shIntentOfClass(t, commit2.History.Intents, "root_sync") + if replacement != secondRoot.ID { + t.Fatalf("first root_sync replacement_intent_id = %q, want %q", replacement, secondRoot.ID) + } + if supersessionReason == "" { + t.Fatal("a superseded root_sync must record why") + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, + sitID); n != 1 { + t.Fatalf("pending root_sync intents = %d, want exactly 1", n) + } +} + +// shMakeDue re-arms situationID's deterministic checkpoint so +// ClaimDueSituations selects it again — the second cycle of a two-cycle +// test, without waiting out a real cadence. +func shMakeDue(t *testing.T, st *Store, situationID string, at time.Time) { + t.Helper() + if _, err := st.db.ExecContext(context.Background(), + `UPDATE situations SET next_assessment_at = ? WHERE id = ?`, canonicalTime(at), situationID); err != nil { + t.Fatalf("re-arm situation %s: %v", situationID, err) + } +} + +func shIntentOfClass(t *testing.T, intents []situationmodel.NotificationIntent, class situationmodel.EffectClass) situationmodel.NotificationIntent { + t.Helper() + for _, i := range intents { + if i.EffectClass == class { + return i + } + } + t.Fatalf("no %s intent among %d planned", class, len(intents)) + return situationmodel.NotificationIntent{} +} + +// TestControllerCommitHistoryRejectsNonContinuingTransitionSequence proves +// the third fence Step 2 names: history derived against a prior Transition +// that is no longer the Situation's current one is rejected whole, rather +// than punching a gap into an immutable ledger. +func TestControllerCommitHistoryRejectsNonContinuingTransitionSequence(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "group-history-sequence-gap", now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + commit := shDerive(t, cycle) + // Pretend the controller derived against a prior Transition at + // sequence 4 while the Situation has none at all. + for i := range commit.History.Transitions { + commit.History.Transitions[i].Sequence += 4 + } + commit.History.Summary.SourceTransitionSequence += 4 + + if err := st.CommitController(ctx, claim, commit); !errors.Is(err, ErrSituationHistoryConflict) { + t.Fatalf("CommitController with a non-continuing sequence = %v, want ErrSituationHistoryConflict", err) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, sitID); n != 0 { + t.Fatalf("transitions after a sequence-conflict rejection = %d, want 0", n) + } +} + +// ---------------------------------------------------------------------- +// Step 5: the coherent load the controller derives history from. +// ---------------------------------------------------------------------- + +// TestLoadReconciliationInputReadsPriorHistoryAndPendingArtifacts proves the +// controller's coherent load returns the prior Transition, the current +// Episode summary, and only the applied-and-unjournaled artifacts, in R1's +// (applied_input_version, occurred_at, id) order. +func TestLoadReconciliationInputReadsPriorHistoryAndPendingArtifacts(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + group := "group-load-history" + sitID := newSituationForGroup(t, st, group, now) + + claim := claimSituation(t, st, sitID, "controller-a", now) + first := shDerive(t, shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now)) + if err := st.CommitController(ctx, claim, first); err != nil { + t.Fatalf("first CommitController: %v", err) + } + + // Two artifacts pending, deliberately seeded out of order. + shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-zz", + "captured_verdict_recorded", claim.Situation.InputVersion, now.Add(-time.Minute)) + shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-aa", + "operator_annotation_recorded", claim.Situation.InputVersion, now.Add(-2*time.Minute)) + + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim2 := claimSituation(t, st, sitID, "controller-a", now.Add(time.Minute)) + in, err := st.LoadReconciliationInput(ctx, claim2, now.Add(time.Minute)) + if err != nil { + t.Fatalf("LoadReconciliationInput: %v", err) + } + if in.PriorTransition == nil || in.PriorTransition.ID != first.History.Transitions[0].ID { + t.Fatalf("prior transition = %+v, want the committed %q", in.PriorTransition, first.History.Transitions[0].ID) + } + if in.CurrentSummary == nil || in.CurrentSummary.Version != 1 { + t.Fatalf("current summary = %+v, want version 1", in.CurrentSummary) + } + if in.RootPublished { + t.Fatal("root_published must stay false until Slack coordinates are durable") + } + if in.LatestRootSyncVersion == nil || *in.LatestRootSyncVersion != 1 { + t.Fatalf("latest root sync version = %v, want 1", in.LatestRootSyncVersion) + } + if in.LastDeliveredRootDeadlineAt != nil || in.LastMainChannelPokeAt != nil { + t.Fatalf("nothing has been delivered yet, got (%v,%v)", in.LastDeliveredRootDeadlineAt, in.LastMainChannelPokeAt) + } + if len(in.PendingArtifacts) != 2 { + t.Fatalf("pending artifacts = %d, want 2", len(in.PendingArtifacts)) + } + if in.PendingArtifacts[0].InputID != "input-aa" || in.PendingArtifacts[1].InputID != "input-zz" { + t.Fatalf("pending artifact order = %q,%q, want input-aa then input-zz (occurred_at order)", + in.PendingArtifacts[0].InputID, in.PendingArtifacts[1].InputID) + } + if in.PendingArtifacts[0].AnnotationID == nil || in.PendingArtifacts[0].Headline == "" { + t.Fatalf("annotation artifact lost its durable content: %+v", in.PendingArtifacts[0]) + } + if in.PendingArtifacts[1].VerdictID == nil || in.PendingArtifacts[1].AttributedActor == "" { + t.Fatalf("verdict artifact lost its durable content: %+v", in.PendingArtifacts[1]) + } +} + +// TestLoadReconciliationInputSkipsJournaledAndOwnerTerminalArtifacts pins R2 +// at the load boundary: an artifact recorded against an already-terminal +// owner is never offered for journaling, and neither is one a previous +// commit already consumed. +func TestLoadReconciliationInputSkipsJournaledAndOwnerTerminalArtifacts(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + group := "group-load-r2" + sitID := newSituationForGroup(t, st, group, now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + pending := shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-live", + "operator_annotation_recorded", claim.Situation.InputVersion, now.Add(-time.Minute)) + late := shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-late", + "operator_annotation_recorded", claim.Situation.InputVersion, now) + if _, err := st.db.ExecContext(ctx, + `UPDATE situation_input_outbox SET journal_state = 'owner_terminal' WHERE id = ?`, late.InputID); err != nil { + t.Fatalf("mark artifact owner_terminal: %v", err) + } + + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + cycle.Change.OperatorArtifacts = []situation.OperatorArtifactInput{pending} + if err := st.CommitController(ctx, claim, shDerive(t, cycle)); err != nil { + t.Fatalf("CommitController: %v", err) + } + + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim2 := claimSituation(t, st, sitID, "controller-a", now.Add(time.Minute)) + in, err := st.LoadReconciliationInput(ctx, claim2, now.Add(time.Minute)) + if err != nil { + t.Fatalf("LoadReconciliationInput: %v", err) + } + if len(in.PendingArtifacts) != 0 { + t.Fatalf("pending artifacts = %+v, want none (one journaled, one owner_terminal)", in.PendingArtifacts) + } + var lateState string + var lateTransition nullStringForTest + if err := st.db.QueryRowContext(ctx, + `SELECT journal_state, journaled_transition_id FROM situation_input_outbox WHERE id = ?`, + late.InputID).Scan(&lateState, &lateTransition); err != nil { + t.Fatalf("read late artifact: %v", err) + } + if lateState != "owner_terminal" || lateTransition.Valid { + t.Fatalf("late artifact = (%q,%v), want owner_terminal with no Transition", lateState, lateTransition) + } +} + +// TestControllerCommitHistoryLeavesEveryEffectRecoverable is the "crash +// after commit" half of the crash boundary: once the fenced transaction +// lands, every outward effect it created is still pending and claimable by +// a restarted worker — nothing was consumed by the commit itself. +func TestControllerCommitHistoryLeavesEveryEffectRecoverable(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "group-history-recoverable", now) + claim := claimSituation(t, st, sitID, "controller-a", now) + + commit := shDerive(t, shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now)) + if err := st.CommitController(ctx, claim, commit); err != nil { + t.Fatalf("CommitController: %v", err) + } + + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM situation_transition_stream WHERE situation_id = ? AND status = 'pending' AND lease_owner IS NULL`, + sitID); n != len(commit.History.Transitions) { + t.Fatalf("recoverable stream rows = %d, want %d", n, len(commit.History.Transitions)) + } + pendingIntents := 0 + for _, intent := range commit.History.Intents { + if intent.Status == situationmodel.IntentPending { + pendingIntents++ + } + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND status = 'pending' AND claim_owner IS NULL AND attempt_count = 0`, + sitID); n != pendingIntents { + t.Fatalf("recoverable pending intents = %d, want %d", n, pendingIntents) + } + // The Situation's own lease is released, so the next claim is free to + // pick the work up again. + var leaseOwner nullStringForTest + if err := st.db.QueryRowContext(ctx, `SELECT lease_owner FROM situations WHERE id = ?`, sitID).Scan(&leaseOwner); err != nil { + t.Fatalf("read lease owner: %v", err) + } + if leaseOwner.Valid { + t.Fatalf("lease owner after commit = %q, want released", leaseOwner.String) + } +} + +// TestControllerCommitHistoryTerminalCycleJournalsArtifactThenCloses is R1's +// terminal ordering against the real schema: an artifact still pending when +// the Situation closes is journaled in the same transaction, before the +// terminal Transition, and the terminal Episode summary lands with it. +func TestControllerCommitHistoryTerminalCycleJournalsArtifactThenCloses(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + group := "group-history-terminal" + sitID := newSituationForGroup(t, st, group, now) + + claim := claimSituation(t, st, sitID, "controller-a", now) + first := shDerive(t, shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now)) + if err := st.CommitController(ctx, claim, first); err != nil { + t.Fatalf("first CommitController: %v", err) + } + + later := now.Add(time.Minute) + artifact := shSeedPendingArtifact(t, st, sitID, "inc-"+group, group, "input-terminal", + "operator_annotation_recorded", claim.Situation.InputVersion, now) + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim2 := claimSituation(t, st, sitID, "controller-a", later) + + terminal := shPrepare(t, claim2, situationmodel.ActionContract{NextActor: situationmodel.NextActorNone}, + situationmodel.LifecycleClosedUnknown, situationmodel.AttentionObserve, later) + terminal.Change.PriorTransition = &first.History.Transitions[0] + terminal.Change.PriorSummary = first.History.Summary + terminal.Change.OperatorArtifacts = []situation.OperatorArtifactInput{artifact} + terminal.Publish.PriorTransition = &first.History.Transitions[0] + terminal.Publish.RootPublished = true + commit := shDerive(t, terminal) + if err := st.CommitController(ctx, claim2, commit); err != nil { + t.Fatalf("terminal CommitController: %v", err) + } + + if got := shTransitionSequences(t, st, sitID); len(got) != 3 { + t.Fatalf("transition sequences = %v, want three (publication, artifact, closure)", got) + } + var reasons []string + rows, err := st.db.QueryContext(ctx, + `SELECT reason FROM situation_transitions WHERE situation_id = ? ORDER BY sequence`, sitID) + if err != nil { + t.Fatalf("read transition reasons: %v", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var reason string + if err := rows.Scan(&reason); err != nil { + t.Fatalf("scan transition reason: %v", err) + } + reasons = append(reasons, reason) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate transition reasons: %v", err) + } + want := []string{"first_authoritative_state", "operator_artifact_recorded", "closed_unknown"} + for i := range want { + if i >= len(reasons) || reasons[i] != want[i] { + t.Fatalf("transition reasons = %v, want %v", reasons, want) + } + } + var state, journaledTransitionID string + if err := st.db.QueryRowContext(ctx, + `SELECT journal_state, journaled_transition_id FROM situation_input_outbox WHERE id = ?`, + artifact.InputID).Scan(&state, &journaledTransitionID); err != nil { + t.Fatalf("read artifact: %v", err) + } + if state != "journaled" || journaledTransitionID != commit.History.Transitions[0].ID { + t.Fatalf("artifact = (%q,%q), want journaled by the artifact Transition %q", + state, journaledTransitionID, commit.History.Transitions[0].ID) + } + view, err := st.GetSituationEpisodeView(ctx, sitID) + if err != nil { + t.Fatalf("GetSituationEpisodeView: %v", err) + } + if view.Summary.Version != 3 || view.Summary.TerminalAt == nil { + t.Fatalf("terminal episode summary = %+v, want version 3 with a terminal instant", view.Summary) + } + if view.SourceTransition.Reason != "closed_unknown" { + t.Fatalf("terminal summary source transition reason = %q, want closed_unknown", view.SourceTransition.Reason) + } + // A terminal root carries no promised update time. + root := shIntentOfClass(t, commit.History.Intents, "root_sync") + stored, err := st.GetNotificationIntent(ctx, root.ID) + if err != nil { + t.Fatalf("GetNotificationIntent: %v", err) + } + if stored.ContractDeadlineAt != nil { + t.Fatalf("terminal root_sync contract deadline = %v, want none", stored.ContractDeadlineAt) + } +} diff --git a/internal/store/situation_views.go b/internal/store/situation_views.go index 94d5b94..b403045 100644 --- a/internal/store/situation_views.go +++ b/internal/store/situation_views.go @@ -342,3 +342,196 @@ func (s *Store) listIncidentTriageViews(ctx context.Context, situationID string) } return out, nil } + +// ---------------------------------------------------------------------- +// Plan 3 Task 5: bounded, coherent history read views. Slack delivery, MCP, +// the stdout stream, and replay all read durable history through these — +// never by joining the ledger ad hoc. Every page takes an explicit limit +// and a stable (sequence, id) cursor, and the Episode view reads its +// summary and that summary's source Transition inside ONE snapshot +// transaction so a caller can never combine a newer summary with a source +// Transition it cannot see. +// ---------------------------------------------------------------------- + +// maxSituationHistoryPage bounds one Transition/stream page. A caller may +// ask for less; it never gets more. +const maxSituationHistoryPage = 100 + +// TransitionCursor is the stable position of one Transition page: resume +// strictly after this (sequence, id). The zero value starts at the +// beginning. +type TransitionCursor struct { + Sequence int + ID string +} + +// SituationEpisodeView is a Situation's current Episode-summary projection +// together with the exact Transition it was folded from — read coherently, +// so the two can never disagree. +type SituationEpisodeView struct { + Summary situationmodel.EpisodeSummary + SourceTransition situationmodel.Transition +} + +// GetSituationEpisodeView reads situationID's current Episode summary and +// its source Transition in one snapshot transaction. Returns ErrNotFound +// when the Situation has no Transition folded yet. +func (s *Store) GetSituationEpisodeView(ctx context.Context, situationID string) (SituationEpisodeView, error) { + if strings.TrimSpace(situationID) == "" { + return SituationEpisodeView{}, errors.New("store: situation episode view requires a situation id") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return SituationEpisodeView{}, fmt.Errorf("store: begin situation episode view: %w", err) + } + defer func() { _ = tx.Rollback() }() + + summary, err := loadEpisodeSummaryTx(ctx, tx, situationID) + if err != nil { + return SituationEpisodeView{}, err + } + if summary == nil { + return SituationEpisodeView{}, ErrNotFound + } + source, err := scanTransition(tx.QueryRowContext(ctx, + `SELECT `+transitionColumns+` FROM situation_transitions WHERE situation_id = ? AND sequence = ?`, + situationID, summary.SourceTransitionSequence)) + if errors.Is(err, sql.ErrNoRows) { + // Unreachable through the fenced commit (the summary's composite + // foreign key names this exact row), so this can only mean a + // hand-edited database — fail closed rather than return a summary + // whose authority is missing. + return SituationEpisodeView{}, fmt.Errorf("store: episode summary for %s names transition sequence %d, which does not exist", + situationID, summary.SourceTransitionSequence) + } + if err != nil { + return SituationEpisodeView{}, fmt.Errorf("store: read episode source transition: %w", err) + } + if err := tx.Commit(); err != nil { + return SituationEpisodeView{}, fmt.Errorf("store: commit situation episode view: %w", err) + } + return SituationEpisodeView{Summary: *summary, SourceTransition: source}, nil +} + +// ListSituationTransitions reads one ordered page of situationID's +// immutable Transition ledger, strictly after cursor, oldest first. limit +// is clamped to maxSituationHistoryPage. +func (s *Store) ListSituationTransitions(ctx context.Context, situationID string, cursor TransitionCursor, limit int) ([]situationmodel.Transition, error) { + if strings.TrimSpace(situationID) == "" { + return nil, errors.New("store: situation transition page requires a situation id") + } + if limit <= 0 || limit > maxSituationHistoryPage { + limit = maxSituationHistoryPage + } + rows, err := s.db.QueryContext(ctx, ` + SELECT `+transitionColumns+` + FROM situation_transitions + WHERE situation_id = ? AND (sequence > ? OR (sequence = ? AND id > ?)) + ORDER BY sequence ASC, id ASC + LIMIT ?`, situationID, cursor.Sequence, cursor.Sequence, cursor.ID, limit) + if err != nil { + return nil, fmt.Errorf("store: list situation transitions: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []situationmodel.Transition{} + for rows.Next() { + tr, err := scanTransition(rows) + if err != nil { + return nil, fmt.Errorf("store: scan situation transition: %w", err) + } + out = append(out, tr) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate situation transitions: %w", err) + } + return out, nil +} + +// GetSituationTransition reads one exact Transition by ID. Returns +// ErrNotFound when no such Transition exists. +func (s *Store) GetSituationTransition(ctx context.Context, transitionID string) (situationmodel.Transition, error) { + if strings.TrimSpace(transitionID) == "" { + return situationmodel.Transition{}, errors.New("store: situation transition read requires a transition id") + } + tr, err := scanTransition(s.db.QueryRowContext(ctx, + `SELECT `+transitionColumns+` FROM situation_transitions WHERE id = ?`, transitionID)) + if errors.Is(err, sql.ErrNoRows) { + return situationmodel.Transition{}, ErrNotFound + } + if err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: read situation transition: %w", err) + } + return tr, nil +} + +// GetNotificationIntent reads one exact durable notification intent by ID. +// Returns ErrNotFound when no such intent exists. +func (s *Store) GetNotificationIntent(ctx context.Context, intentID string) (situationmodel.NotificationIntent, error) { + if strings.TrimSpace(intentID) == "" { + return situationmodel.NotificationIntent{}, errors.New("store: notification intent read requires an intent id") + } + intent, err := scanNotificationIntent(s.db.QueryRowContext(ctx, + `SELECT `+notificationIntentColumns+` FROM notification_intents WHERE id = ?`, intentID)) + if errors.Is(err, sql.ErrNoRows) { + return situationmodel.NotificationIntent{}, ErrNotFound + } + if err != nil { + return situationmodel.NotificationIntent{}, fmt.Errorf("store: read notification intent: %w", err) + } + return intent, nil +} + +// PendingTransitionStreamEntry is one undelivered stdout-stream row plus the +// immutable Transition it records — everything the stdout writer needs +// without a second lookup. +type PendingTransitionStreamEntry struct { + StreamID string + Transition situationmodel.Transition +} + +// ListPendingTransitionStream reads one ordered page of undelivered +// stdout-stream rows across every Situation, oldest first, joined to their +// Transitions in one snapshot transaction so an entry can never name a +// Transition the same read cannot see. limit is clamped to +// maxSituationHistoryPage. +func (s *Store) ListPendingTransitionStream(ctx context.Context, limit int) ([]PendingTransitionStreamEntry, error) { + if limit <= 0 || limit > maxSituationHistoryPage { + limit = maxSituationHistoryPage + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("store: begin pending transition stream page: %w", err) + } + defer func() { _ = tx.Rollback() }() + + rows, err := tx.QueryContext(ctx, ` + SELECT st.id, `+prefixedTransitionColumns+` + FROM situation_transition_stream st + JOIN situation_transitions t ON t.id = st.transition_id + WHERE st.status = 'pending' + ORDER BY st.created_at ASC, st.situation_id ASC, st.sequence ASC + LIMIT ?`, limit) + if err != nil { + return nil, fmt.Errorf("store: list pending transition stream: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []PendingTransitionStreamEntry{} + for rows.Next() { + var entry PendingTransitionStreamEntry + tr, err := scanStreamEntry(rows, &entry.StreamID) + if err != nil { + return nil, err + } + entry.Transition = tr + out = append(out, entry) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate pending transition stream: %w", err) + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("store: commit pending transition stream page: %w", err) + } + return out, nil +} diff --git a/internal/store/situation_views_test.go b/internal/store/situation_views_test.go index 0885268..ceb6479 100644 --- a/internal/store/situation_views_test.go +++ b/internal/store/situation_views_test.go @@ -474,3 +474,162 @@ func TestSituationControllerViewUnknownSituationReturnsErrNotFound(t *testing.T) t.Fatalf("err = %v, want ErrNotFound", err) } } + +// ---------------------------------------------------------------------- +// Plan 3 Task 5: bounded history read views. +// ---------------------------------------------------------------------- + +// shSeedTwoCommitHistory lands two real fenced commits for one Situation and +// returns its id plus the second commit, so the history views below read +// genuinely committed durable records rather than hand-inserted rows. +func shSeedTwoCommitHistory(t *testing.T, st *Store, group string, now time.Time) (string, situation.ControllerCommit) { + t.Helper() + ctx := context.Background() + sitID := newSituationForGroup(t, st, group, now) + + claim := claimSituation(t, st, sitID, "controller-a", now) + first := shDerive(t, shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now)) + if err := st.CommitController(ctx, claim, first); err != nil { + t.Fatalf("first CommitController: %v", err) + } + + later := now.Add(time.Minute) + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim2 := claimSituation(t, st, sitID, "controller-a", later) + cycle2 := shPrepare(t, claim2, shOperatorContract(later.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, later) + cycle2.Change.PriorTransition = &first.History.Transitions[0] + cycle2.Change.PriorSummary = first.History.Summary + cycle2.Publish.PriorTransition = &first.History.Transitions[0] + cycle2.Publish.RootPublished = true + second := shDerive(t, cycle2) + if err := st.CommitController(ctx, claim2, second); err != nil { + t.Fatalf("second CommitController: %v", err) + } + return sitID, second +} + +func TestSituationHistoryViewReadsCurrentEpisodeWithItsSourceTransition(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, second := shSeedTwoCommitHistory(t, st, "group-view-episode", now) + + view, err := st.GetSituationEpisodeView(context.Background(), sitID) + if err != nil { + t.Fatalf("GetSituationEpisodeView: %v", err) + } + if view.Summary.Version != 2 { + t.Fatalf("summary version = %d, want 2", view.Summary.Version) + } + want := second.History.Transitions[0] + if view.SourceTransition.ID != want.ID { + t.Fatalf("source transition = %q, want %q", view.SourceTransition.ID, want.ID) + } + if view.Summary.SourceTransitionSequence != view.SourceTransition.Sequence { + t.Fatalf("summary source sequence %d does not match the returned transition sequence %d", + view.Summary.SourceTransitionSequence, view.SourceTransition.Sequence) + } + if view.SourceTransition.Reason != want.Reason || view.SourceTransition.JournalKind != want.JournalKind { + t.Fatalf("source transition round trip lost typed fields: %+v", view.SourceTransition) + } +} + +func TestSituationHistoryViewUnknownSituationEpisodeReturnsErrNotFound(t *testing.T) { + st := newTestStore(t) + if _, err := st.GetSituationEpisodeView(context.Background(), "no-such-situation"); !errors.Is(err, ErrNotFound) { + t.Fatalf("GetSituationEpisodeView(unknown) = %v, want ErrNotFound", err) + } +} + +func TestSituationHistoryViewPagesTransitionsByStableCursorAndLimit(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := shSeedTwoCommitHistory(t, st, "group-view-page", now) + + page, err := st.ListSituationTransitions(ctx, sitID, TransitionCursor{}, 1) + if err != nil { + t.Fatalf("ListSituationTransitions(first page): %v", err) + } + if len(page) != 1 || page[0].Sequence != 1 { + t.Fatalf("first page = %d transitions (%+v), want exactly sequence 1", len(page), page) + } + next, err := st.ListSituationTransitions(ctx, sitID, TransitionCursor{Sequence: page[0].Sequence, ID: page[0].ID}, 10) + if err != nil { + t.Fatalf("ListSituationTransitions(second page): %v", err) + } + if len(next) != 1 || next[0].Sequence != 2 { + t.Fatalf("second page = %+v, want exactly sequence 2", next) + } + tail, err := st.ListSituationTransitions(ctx, sitID, TransitionCursor{Sequence: next[0].Sequence, ID: next[0].ID}, 10) + if err != nil { + t.Fatalf("ListSituationTransitions(tail): %v", err) + } + if len(tail) != 0 { + t.Fatalf("tail page = %+v, want empty", tail) + } +} + +func TestSituationHistoryViewReadsExactTransitionAndIntentByID(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, second := shSeedTwoCommitHistory(t, st, "group-view-exact", now) + + want := second.History.Transitions[0] + got, err := st.GetSituationTransition(ctx, want.ID) + if err != nil { + t.Fatalf("GetSituationTransition: %v", err) + } + if got.ID != want.ID || got.Sequence != want.Sequence || got.Journal.Headline != want.Journal.Headline { + t.Fatalf("GetSituationTransition = %+v, want the committed %+v", got, want) + } + if _, err := st.GetSituationTransition(ctx, "no-such-transition"); !errors.Is(err, ErrNotFound) { + t.Fatalf("GetSituationTransition(unknown) = %v, want ErrNotFound", err) + } + + wantIntent := shIntentOfClass(t, second.History.Intents, "root_sync") + gotIntent, err := st.GetNotificationIntent(ctx, wantIntent.ID) + if err != nil { + t.Fatalf("GetNotificationIntent: %v", err) + } + if gotIntent.IdempotencyKey != wantIntent.IdempotencyKey || gotIntent.EffectClass != wantIntent.EffectClass { + t.Fatalf("GetNotificationIntent = %+v, want the committed %+v", gotIntent, wantIntent) + } + if gotIntent.SummaryVersion == nil || *gotIntent.SummaryVersion != *wantIntent.SummaryVersion { + t.Fatalf("root_sync summary version round trip = %v, want %v", gotIntent.SummaryVersion, wantIntent.SummaryVersion) + } + if _, err := st.GetNotificationIntent(ctx, "no-such-intent"); !errors.Is(err, ErrNotFound) { + t.Fatalf("GetNotificationIntent(unknown) = %v, want ErrNotFound", err) + } +} + +func TestSituationHistoryViewPendingStdoutPageJoinsItsTransition(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := shSeedTwoCommitHistory(t, st, "group-view-stream", now) + + page, err := st.ListPendingTransitionStream(ctx, 10) + if err != nil { + t.Fatalf("ListPendingTransitionStream: %v", err) + } + mine := []PendingTransitionStreamEntry{} + for _, e := range page { + if e.Transition.SituationID == sitID { + mine = append(mine, e) + } + } + if len(mine) != 2 { + t.Fatalf("pending stream entries for %s = %d, want 2", sitID, len(mine)) + } + for i, e := range mine { + if e.StreamID == "" { + t.Fatalf("entry %d carries no stream id", i) + } + if e.Transition.Sequence != i+1 { + t.Fatalf("entry %d transition sequence = %d, want %d", i, e.Transition.Sequence, i+1) + } + } +} From 2a53f6000a5124ca0112bab9ad0c81f8db77772f Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 03:01:34 +0300 Subject: [PATCH 08/31] fix(situation): make the terminal-fold exception commit-identifying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on 21a5b30. - validateFold's "same terminal commit" test compared only Projection.TerminalAt, which resolveLifecycle carries forward unchanged on every later cycle — it identifies a closure, not a write. A future reconciliation of an already-terminal Situation (migration 0017's header contemplates one: upgrade_reconstruction for pre-ledger Plan 1/2 Situations) would have reused the same instant and been let through, leaving "a terminal Episode never reopens" enforced only by ClaimDueSituations' lifecycle filter rather than by ProjectEpisode itself. The exception now also requires t.CreatedAt == prior.UpdatedAt: every Transition of one commit is stamped with that reconciliation's single Now, and each fold copies it onto the summary, so this holds for the rest of this commit and for nothing later. - supersedePendingRootSyncTx clears the superseded intent's claim fields (0018's claim_owner/status CHECK requires it). That consequence lived only in the task report, not where the notification worker's implementer will read it: a worker can lose a live claim to a concurrent controller commit and must re-check status before writing a delivery outcome, since a superseded intent can never reach 'delivered'. Documented on the function. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/situation/history.go | 25 ++++++++++++++++++------- internal/situation/history_test.go | 16 ++++++++++++++++ internal/store/situation_history.go | 15 +++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/internal/situation/history.go b/internal/situation/history.go index fef3dbf..5711f42 100644 --- a/internal/situation/history.go +++ b/internal/situation/history.go @@ -863,15 +863,26 @@ func ProjectEpisode(prior *model.EpisodeSummary, t model.Transition) (model.Epis // controller-state Transition, and every Transition of one commit carries // the same captured projection (R3) — so a commit that both journals an // artifact and closes the Situation folds an artifact Transition that -// already reports the closure, then the terminal Transition itself. A -// Transition reporting the identical terminal instant is by construction -// part of the commit that closed the Episode, never a later reopening: a -// terminal Situation is never claimed again (ClaimDueSituations selects -// only active/recovery_pending) and a later artifact is recorded, never -// journaled (R2). +// already reports the closure, then the terminal Transition itself. +// +// "Same commit" is established by TWO facts together, because neither alone +// identifies a commit. The terminal instant is a DURABLE value that +// resolveLifecycle carries forward unchanged on every later cycle, so a +// matching Projection.TerminalAt proves only "the same closure", not "the +// same write". The commit identity is the instant: every Transition of one +// commit is stamped with that reconciliation's single Now (newTransition), +// and each fold copies it onto the summary as UpdatedAt — so +// t.CreatedAt.Equal(prior.UpdatedAt) holds for the rest of this commit and +// for nothing later. Requiring both keeps "a terminal Episode never +// reopens" an invariant of THIS function rather than something only +// ClaimDueSituations' active/recovery_pending filter happens to prevent — +// which matters for any future path that reconciles an already-terminal +// Situation (migration 0017's own header contemplates one: an idempotent +// upgrade_reconstruction for Plan 1/2 Situations that predate the ledger). func validateFold(prior model.EpisodeSummary, t model.Transition) error { sameTerminalCommit := prior.TerminalAt != nil && t.Projection.TerminalAt != nil && - t.Projection.TerminalAt.Equal(*prior.TerminalAt) + t.Projection.TerminalAt.Equal(*prior.TerminalAt) && + t.CreatedAt.Equal(prior.UpdatedAt) switch { case prior.SituationID != t.SituationID: return fmt.Errorf("situation: project episode: transition belongs to situation %q, summary to %q", diff --git a/internal/situation/history_test.go b/internal/situation/history_test.go index 69f4279..ac6674f 100644 --- a/internal/situation/history_test.go +++ b/internal/situation/history_test.go @@ -1503,4 +1503,20 @@ func TestProjectEpisodeFoldsTheRestOfOneTerminalCommitButNeverReopens(t *testing if _, err := ProjectEpisode(commit.Summary, reopening); err == nil { t.Fatal("a nonterminal Transition must never reopen a terminal Episode") } + + // And so does a LATER commit that carries the SAME terminal instant — + // which is exactly what a future reconciliation of an already-terminal + // Situation would look like, since resolveLifecycle carries terminal_at + // forward unchanged. The terminal instant alone never identifies a + // commit; only the reconciliation instant does. + sameClosureLater := commit.Transitions[1] + sameClosureLater.Sequence++ + sameClosureLater.CreatedAt = commit.Summary.UpdatedAt.Add(time.Hour) + if sameClosureLater.Projection.TerminalAt == nil || + !sameClosureLater.Projection.TerminalAt.Equal(*commit.Summary.TerminalAt) { + t.Fatalf("fixture must reuse the committed terminal instant, got %v", sameClosureLater.Projection.TerminalAt) + } + if _, err := ProjectEpisode(commit.Summary, sameClosureLater); err == nil { + t.Fatal("a later commit reusing the same terminal instant must never fold onto a terminal Episode") + } } diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go index 36cd61e..0eb6215 100644 --- a/internal/store/situation_history.go +++ b/internal/store/situation_history.go @@ -498,6 +498,21 @@ func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID st // and the "at most one pending root_sync" index would otherwise make the // two writes impossible to order. Deferral changes when a violation is // reported, never whether the transaction is atomic. +// +// FOR THE NOTIFICATION WORKER: superseding CLEARS the intent's +// claim_owner/lease_expires_at (migration 0018's +// `claim_owner IS NULL OR status = 'pending'` CHECK forbids leaving them on +// a non-pending row) and its retry_at. A worker holding a live claim on a +// root_sync can therefore have that claim taken out from under it by a +// concurrent controller commit, and must re-read the intent's status before +// writing any delivery outcome: a superseded row can never become +// 'delivered', because 0018's +// `CHECK ((status = 'superseded') = (supersession_reason IS NOT NULL))` +// aborts that write. Treat the lost claim as the expected R4 outcome — the +// newer root projection supersedes what this one would have posted — not as +// a delivery failure. Supersession is performed here, inside the +// authoritative commit, precisely because the pending-root index makes it +// unorderable anywhere else; the worker must not reimplement it. func supersedePendingRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, replacementID string) error { var pending int if err := tx.QueryRowContext(ctx, ` From 2028214e93216600c1d7623444dda71c6e847f46 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 03:50:05 +0300 Subject: [PATCH 09/31] feat(slack): render Situation roots and journals Pure Situation-owned Slack rendering (roots, immutable journal entries, the installation Delivery-gap notice), a narrow hand-rolled Slack Web API client (chat.postMessage/chat.update/auth.test) with typed retryable/configuration-blocking/invalid error classification, and the concrete deliverer adapter that loads durable state through Task 5's readers and makes exactly one Slack call per notification intent. No publication decision and no Store write live in this layer. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- cmd/alertint/situation_notifications.go | 368 +++++++++ cmd/alertint/situation_notifications_test.go | 695 ++++++++++++++++ internal/notify/slack/api_client.go | 405 +++++++++ internal/notify/slack/api_client_test.go | 500 ++++++++++++ internal/notify/slack/situation.go | 675 +++++++++++++++ internal/notify/slack/situation_test.go | 816 +++++++++++++++++++ internal/notify/slack/time.go | 48 ++ internal/notify/slack/time_test.go | 95 +++ 8 files changed, 3602 insertions(+) create mode 100644 cmd/alertint/situation_notifications.go create mode 100644 cmd/alertint/situation_notifications_test.go create mode 100644 internal/notify/slack/api_client.go create mode 100644 internal/notify/slack/api_client_test.go create mode 100644 internal/notify/slack/situation.go create mode 100644 internal/notify/slack/situation_test.go create mode 100644 internal/notify/slack/time.go create mode 100644 internal/notify/slack/time_test.go diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go new file mode 100644 index 0000000..a33d436 --- /dev/null +++ b/cmd/alertint/situation_notifications.go @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/alertint/alertint-agent/internal/notify/slack" + "github.com/alertint/alertint-agent/internal/situation/model" + "github.com/alertint/alertint-agent/internal/store" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 6: the concrete Slack deliverer adapter. It makes no +// publication decision (that authority lives entirely in +// internal/situation's BuildTransitions/PlanNotificationIntents, Task 4) +// and never writes Store state itself — given one already-selected +// model.NotificationIntent, it loads the exact durable records it +// references through Task 5's bounded Store readers, renders them (pure +// internal/notify/slack functions), makes exactly one Slack call, and +// returns the result. Claim/lease ownership, retry/backoff, Delivery-gap +// tracking, and acknowledging the result back into the Store are Task 7's +// notification worker, not this file. +// +// NotificationDelivery below is a TEMPORARY stand-in. The plan's +// Cross-Task Contracts define `NotificationDelivery{Channel, MessageTS, +// DeliveredAs}` and `NotificationDeliverer{Probe, Deliver}` in +// internal/situation/notification_worker.go — which is Task 7's file and +// does not exist yet. This type carries EXACTLY those field names and +// shapes so Task 7's dispatch can replace this file's `NotificationDelivery` +// references with `situation.NotificationDelivery` (and delete this local +// definition) as a mechanical rename, with no other change to +// SituationDeliverer's logic expected. +// ---------------------------------------------------------------------- + +// NotificationDelivery is the durable Slack coordinate and delivery shape +// one Deliver call returns. DeliveredAs is one of: root | thread | +// broadcast | delayed_thread | system. +type NotificationDelivery struct { + Channel string + MessageTS string + DeliveredAs string +} + +// GapSnapshot is the durable installation-level Delivery-gap generation +// RenderDeliveryGapNotice renders one recovery notice from: opened_at, +// recovered_at, and the affected/delayed counts recorded on +// slack_delivery_gaps (migration 0018). No Task 5 reader exposes this +// table today (Task 5's situation_views.go covers Situation/Transition/ +// intent reads only); Task 7's internal/store/notification_gaps.go is +// expected to add a matching GetDeliveryGap(ctx, id) (GapSnapshot, error) +// reader on *store.Store so it satisfies DelivererStore unchanged. +type GapSnapshot struct { + ID string + OpenedAt time.Time + RecoveredAt time.Time + AffectedSituationCount int + DelayedEffectCount int +} + +// DelivererStore is exactly what SituationDeliverer reads. The first three +// methods are Task 5's existing bounded readers +// (internal/store/situation_views.go) — *store.Store already satisfies +// them. The last two are new readers this task's design found missing (see +// their own doc comments): Task 7 (or a follow-up to Task 5) is expected to +// add them to *store.Store with exactly these signatures so *store.Store +// satisfies DelivererStore unchanged; this task's own tests exercise +// SituationDeliverer against a hand-rolled fake instead. +type DelivererStore interface { + GetSituationEpisodeView(ctx context.Context, situationID string) (store.SituationEpisodeView, error) + GetSituationTransition(ctx context.Context, transitionID string) (model.Transition, error) + ListSituationTransitions(ctx context.Context, situationID string, cursor store.TransitionCursor, limit int) ([]model.Transition, error) + + // GetSituationRootCoordinates reads the Situation's current durable + // root coordinates (migration 0018's situations.slack_channel / + // slack_root_ts). ok is false when no root has been delivered yet. + GetSituationRootCoordinates(ctx context.Context, situationID string) (channel, messageTS string, ok bool, err error) + + // GetDeliveryGap reads one durable gap generation's rendering facts. + GetDeliveryGap(ctx context.Context, gapGeneration string) (GapSnapshot, error) +} + +// slackDeliveryAPI is exactly what SituationDeliverer calls on the narrow +// Slack Web API client (internal/notify/slack.Client already satisfies +// it). Narrowed to an interface so tests can inject a fake without an +// httptest.Server. +type slackDeliveryAPI interface { + PostMessage(ctx context.Context, req slack.PostMessageRequest) (slack.MessageResult, error) + UpdateMessage(ctx context.Context, req slack.UpdateMessageRequest) (slack.MessageResult, error) + AuthTest(ctx context.Context) error +} + +// maxLedgerScanPages bounds SituationDeliverer's own defensive scan of a +// Situation's Transition ledger (recoveryEverObserved) — a safety cap, not +// a realistic ceiling: a Situation's material Transition count is bounded +// by how many times its state actually changed, not by wall-clock time. +const maxLedgerScanPages = 1000 + +// SituationDeliverer is the concrete Slack NotificationDeliverer adapter +// (R7, Task 6). It reads durable state and calls Slack; it never writes +// Store state. +type SituationDeliverer struct { + store DelivererStore + api slackDeliveryAPI + channel string + now func() time.Time +} + +// NewSituationDeliverer constructs a SituationDeliverer. channel is the +// resolved Situation Slack channel (config.NotifyConfig.Slack.Channel); +// now defaults to time.Now when nil. +func NewSituationDeliverer(delivererStore DelivererStore, api slackDeliveryAPI, channel string, now func() time.Time) *SituationDeliverer { + if now == nil { + now = time.Now + } + return &SituationDeliverer{store: delivererStore, api: api, channel: channel, now: now} +} + +// Probe verifies Slack readiness (auth.test) — the readiness check Task 7's +// gap lifecycle drives before recovery replay and before reactivating +// configuration-blocked intents. +func (d *SituationDeliverer) Probe(ctx context.Context) error { + return d.api.AuthTest(ctx) +} + +// Deliver renders and sends exactly one Slack call for intent, loading the +// exact durable records it references through DelivererStore. It never +// decides whether a root is durably delivered before a reply is claimable +// (Task 7's ordering) and never writes Store state itself. +func (d *SituationDeliverer) Deliver(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { + if err := intent.Validate(); err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: %w", err) + } + switch intent.EffectClass { + case model.EffectRootSync: + return d.deliverRootSync(ctx, intent) + case model.EffectThreadAppend: + return d.deliverThreadAppend(ctx, intent) + case model.EffectBroadcastHandoff: + return d.deliverBroadcastHandoff(ctx, intent) + case model.EffectInstallationGapRecovery: + return d.deliverGapRecovery(ctx, intent) + default: + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: unknown effect class %q", intent.EffectClass) + } +} + +// deliverRootSync posts a first root or edits an existing one in place +// (including the R4 deadline refresh, which is a plain chat.update): the +// selected Episode-summary version renders only from GetSituationEpisodeView's +// own coherent (summary, source Transition) pair. +func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { + if intent.SituationID == nil || intent.SummaryVersion == nil { + return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: root_sync intent missing situation_id/summary_version") + } + view, err := d.store.GetSituationEpisodeView(ctx, *intent.SituationID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) + } + if view.Summary.Version != *intent.SummaryVersion { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: intent names summary version %d, current is %d", + *intent.SummaryVersion, view.Summary.Version) + } + + recoveryEverObserved, err := d.recoveryEverObserved(ctx, view) + if err != nil { + return NotificationDelivery{}, err + } + + rendered, err := slack.RenderSituationRoot(slack.SituationRootInput{ + Summary: view.Summary, + SourceTransition: view.SourceTransition, + ContractDeadlineAt: intent.ContractDeadlineAt, + Now: d.now().UTC(), + RecoveryEverObserved: recoveryEverObserved, + }) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render root: %w", err) + } + + channel, ts, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + } + if !ok { + res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ + Channel: d.channel, + Text: rendered.Text, + Blocks: rendered.Blocks, + ClientMsgID: intent.ClientMessageID, + }) + if err != nil { + return NotificationDelivery{}, err + } + return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "root"}, nil + } + res, err := d.api.UpdateMessage(ctx, slack.UpdateMessageRequest{ + Channel: channel, + TS: ts, + Text: rendered.Text, + Blocks: rendered.Blocks, + ClientMsgID: intent.ClientMessageID, + }) + if err != nil { + return NotificationDelivery{}, err + } + return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "root"}, nil +} + +// deliverThreadAppend appends one immutable journal entry to the +// Situation's existing root thread, rendering only from its own referenced +// Transition. +func (d *SituationDeliverer) deliverThreadAppend(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { + if intent.SituationID == nil || intent.TransitionID == nil { + return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: thread_append intent missing situation_id/transition_id") + } + tr, err := d.store.GetSituationTransition(ctx, *intent.TransitionID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) + } + channel, rootTS, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + } + if !ok { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) + } + rendered, err := slack.RenderSituationJournal(tr) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render journal: %w", err) + } + res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ + Channel: channel, + Text: rendered.Text, + Blocks: rendered.Blocks, + ThreadTS: rootTS, + ClientMsgID: intent.ClientMessageID, + }) + if err != nil { + return NotificationDelivery{}, err + } + return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "thread"}, nil +} + +// deliverBroadcastHandoff optionally broadcasts a current handoff. +// Immediately before external I/O it reloads whether the referenced +// Transition is still the Situation's latest (spec.md "Recovery replay": +// "Immediately before external I/O, reload handoff relevance"); when a +// newer Transition has since superseded it, the same Transition is +// delivered instead as a plain, delayed, no-longer-current thread reply — +// never a channel broadcast. +func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { + if intent.SituationID == nil || intent.TransitionID == nil { + return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: broadcast_handoff intent missing situation_id/transition_id") + } + tr, err := d.store.GetSituationTransition(ctx, *intent.TransitionID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) + } + channel, rootTS, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + } + if !ok { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) + } + view, err := d.store.GetSituationEpisodeView(ctx, *intent.SituationID) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) + } + current := view.Summary.SourceTransitionSequence == tr.Sequence + + renderTr := tr // a local copy: the durable ledger row is never mutated. + if !current { + renderTr.Journal.Delayed = true + renderTr.Journal.NoLongerCurrent = true + } + rendered, err := slack.RenderSituationJournal(renderTr) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render journal: %w", err) + } + + res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ + Channel: channel, + Text: rendered.Text, + Blocks: rendered.Blocks, + ThreadTS: rootTS, + ReplyBroadcast: current, + ClientMsgID: intent.ClientMessageID, + }) + if err != nil { + return NotificationDelivery{}, err + } + deliveredAs := "broadcast" + if !current { + deliveredAs = "delayed_thread" + } + return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: deliveredAs}, nil +} + +// deliverGapRecovery posts the one bounded installation recovery notice for +// gap generation intent.GapGeneration names, rendering only from that +// generation's own durable facts. +func (d *SituationDeliverer) deliverGapRecovery(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { + if intent.GapGeneration == nil { + return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: installation_gap_recovery intent missing gap_generation") + } + gap, err := d.store.GetDeliveryGap(ctx, *intent.GapGeneration) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load delivery gap: %w", err) + } + rendered, err := slack.RenderDeliveryGapNotice(slack.GapNoticeInput{ + GapID: gap.ID, + OpenedAt: gap.OpenedAt, + RecoveredAt: gap.RecoveredAt, + AffectedSituationCount: gap.AffectedSituationCount, + DelayedEffectCount: gap.DelayedEffectCount, + }) + if err != nil { + return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render gap notice: %w", err) + } + res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ + Channel: d.channel, + Text: rendered.Text, + Blocks: rendered.Blocks, + ClientMsgID: intent.ClientMessageID, + }) + if err != nil { + return NotificationDelivery{}, err + } + return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "system"}, nil +} + +// recoveryEverObserved answers SituationRootInput.RecoveryEverObserved: it +// only ever needs the ledger scan for a closed_unknown source Transition +// whose OWN projection carries no recovery observation (see situation.go's +// doc comment on SituationRootInput for why every other case is decided +// without one). +func (d *SituationDeliverer) recoveryEverObserved(ctx context.Context, view store.SituationEpisodeView) (bool, error) { + if view.SourceTransition.Lifecycle != model.LifecycleClosedUnknown { + return false, nil + } + if view.SourceTransition.Projection.RecoveryObservedAt != nil { + return true, nil + } + cursor := store.TransitionCursor{} + for page := 0; page < maxLedgerScanPages; page++ { + transitions, err := d.store.ListSituationTransitions(ctx, view.Summary.SituationID, cursor, 0) + if err != nil { + return false, fmt.Errorf("cmd/alertint: situation deliverer: scan transition ledger: %w", err) + } + if len(transitions) == 0 { + return false, nil + } + for _, tr := range transitions { + if tr.Reason == model.ReasonRecoveryObserved { + return true, nil + } + } + last := transitions[len(transitions)-1] + cursor = store.TransitionCursor{Sequence: last.Sequence, ID: last.ID} + } + return false, fmt.Errorf("cmd/alertint: situation deliverer: transition ledger for %s exceeds %d pages", + view.Summary.SituationID, maxLedgerScanPages) +} diff --git a/cmd/alertint/situation_notifications_test.go b/cmd/alertint/situation_notifications_test.go new file mode 100644 index 0000000..ee24ba5 --- /dev/null +++ b/cmd/alertint/situation_notifications_test.go @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + slacklib "github.com/slack-go/slack" + + "github.com/alertint/alertint-agent/internal/notify/slack" + "github.com/alertint/alertint-agent/internal/situation/model" + "github.com/alertint/alertint-agent/internal/store" +) + +// ---------------------------------------------------------------------- +// Fixtures +// ---------------------------------------------------------------------- + +const sdSituationID = "1f0f5a0c-0000-4000-8000-0000000000d1" + +func sdMustTime(t *testing.T, s string) time.Time { + t.Helper() + tm, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatalf("parse time %q: %v", s, err) + } + return tm.UTC() +} + +func sdTimePtr(t time.Time) *time.Time { return &t } + +func sdRunningTriageContract(next time.Time) model.ActionContract { + action := model.AlertINTActionRunAcuteTriage + status := model.AlertINTStatusRunning + return model.ActionContract{ + NextActor: model.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: sdTimePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnTriageOutcome}, + } +} + +func sdTerminalContract() model.ActionContract { + return model.ActionContract{NextActor: model.NextActorNone} +} + +func sdTransition(seq int, lifecycle model.Lifecycle, contract model.ActionContract, + reason model.TransitionReason, journalKind model.JournalKind, journal model.JournalData, + projection model.ProjectionFacts, createdAt time.Time) model.Transition { + return model.Transition{ + ID: fmt.Sprintf("transition-%03d", seq), + SituationID: sdSituationID, + Sequence: seq, + InputVersion: seq, + MaterialFactHash: "sha256:abc123", + Lifecycle: lifecycle, + Attention: model.AttentionInvestigate, + ActionContract: contract, + Reason: reason, + JournalKind: journalKind, + Journal: journal, + Projection: projection, + EvidenceRefs: []string{"evidence-1"}, + Actor: model.ActorDeterministicController, + CreatedAt: createdAt, + } +} + +func sdSummary(seq int, contract model.ActionContract, startedAt, updatedAt time.Time) model.EpisodeSummary { + return model.EpisodeSummary{ + SituationID: sdSituationID, + Version: seq, + SourceTransitionSequence: seq, + Title: "Situation checkout-001", + CurrentAttention: model.AttentionInvestigate, + PeakAttention: model.AttentionInvestigate, + ActionContract: contract, + EffectiveStartedAt: startedAt, + UpdatedAt: updatedAt, + InvestigationWork: []string{}, + RecordedOperatorContext: []string{}, + } +} + +func sdRootSyncIntent(transitionID string, summaryVersion int, deadline *time.Time, now time.Time) model.NotificationIntent { + situationID := sdSituationID + return model.NotificationIntent{ + ID: "intent-root-1", + IdempotencyKey: "root_sync:1", + EffectClass: model.EffectRootSync, + SituationID: &situationID, + TransitionID: &transitionID, + SummaryVersion: &summaryVersion, + ClientMessageID: "client-msg-root-1", + Status: model.IntentPending, + CreatedAt: now, + ContractDeadlineAt: deadline, + } +} + +func sdThreadIntent(class model.EffectClass, transitionID string, seq int, now time.Time) model.NotificationIntent { + s := seq + situationID := sdSituationID + return model.NotificationIntent{ + ID: "intent-thread-1", + IdempotencyKey: "thread:1", + EffectClass: class, + SituationID: &situationID, + TransitionID: &transitionID, + TransitionSequence: &s, + RequiresRoot: true, + ClientMessageID: "client-msg-thread-1", + Status: model.IntentPending, + CreatedAt: now, + } +} + +func sdGapIntent(gapGeneration string, now time.Time) model.NotificationIntent { + return model.NotificationIntent{ + ID: "intent-gap-1", + IdempotencyKey: "gap:1", + EffectClass: model.EffectInstallationGapRecovery, + GapGeneration: &gapGeneration, + ClientMessageID: "client-msg-gap-1", + Status: model.IntentPending, + CreatedAt: now, + } +} + +// sdBlocksText flattens a rendered Block Kit body's text content for +// substring assertions, mirroring internal/notify/slack's own test helper. +func sdBlocksText(t *testing.T, blocks []slacklib.Block) string { + t.Helper() + var b strings.Builder + for _, blk := range blocks { + switch v := blk.(type) { + case *slacklib.SectionBlock: + if v.Text != nil { + b.WriteString(v.Text.Text) + b.WriteString("\n") + } + case *slacklib.ContextBlock: + for _, el := range v.ContextElements.Elements { + if txt, ok := el.(*slacklib.TextBlockObject); ok { + b.WriteString(txt.Text) + b.WriteString("\n") + } + } + } + } + return b.String() +} + +// ---------------------------------------------------------------------- +// Fakes +// ---------------------------------------------------------------------- + +type fakeDelivererStore struct { + episode store.SituationEpisodeView + episodeErr error + + transitions map[string]model.Transition + transitionErr error + + ledger []model.Transition + ledgerErr error + listCalls int + + rootChannel string + rootTS string + rootOK bool + rootErr error + + gap GapSnapshot + gapErr error +} + +func (f *fakeDelivererStore) GetSituationEpisodeView(context.Context, string) (store.SituationEpisodeView, error) { + if f.episodeErr != nil { + return store.SituationEpisodeView{}, f.episodeErr + } + return f.episode, nil +} + +func (f *fakeDelivererStore) GetSituationTransition(_ context.Context, transitionID string) (model.Transition, error) { + if f.transitionErr != nil { + return model.Transition{}, f.transitionErr + } + tr, ok := f.transitions[transitionID] + if !ok { + return model.Transition{}, store.ErrNotFound + } + return tr, nil +} + +func (f *fakeDelivererStore) ListSituationTransitions(_ context.Context, _ string, cursor store.TransitionCursor, _ int) ([]model.Transition, error) { + f.listCalls++ + if f.ledgerErr != nil { + return nil, f.ledgerErr + } + var out []model.Transition + for _, tr := range f.ledger { + if tr.Sequence > cursor.Sequence || (tr.Sequence == cursor.Sequence && tr.ID > cursor.ID) { + out = append(out, tr) + } + } + return out, nil +} + +func (f *fakeDelivererStore) GetSituationRootCoordinates(context.Context, string) (string, string, bool, error) { + if f.rootErr != nil { + return "", "", false, f.rootErr + } + return f.rootChannel, f.rootTS, f.rootOK, nil +} + +func (f *fakeDelivererStore) GetDeliveryGap(context.Context, string) (GapSnapshot, error) { + if f.gapErr != nil { + return GapSnapshot{}, f.gapErr + } + return f.gap, nil +} + +type fakeSlackAPI struct { + postErr error + updateErr error + authErr error + posts []slack.PostMessageRequest + updates []slack.UpdateMessageRequest + postResult slack.MessageResult + updateResult slack.MessageResult + authCalls int +} + +func (f *fakeSlackAPI) PostMessage(_ context.Context, req slack.PostMessageRequest) (slack.MessageResult, error) { + f.posts = append(f.posts, req) + if f.postErr != nil { + return slack.MessageResult{}, f.postErr + } + if f.postResult != (slack.MessageResult{}) { + return f.postResult, nil + } + return slack.MessageResult{Channel: "C-posted", TS: "9.9"}, nil +} + +func (f *fakeSlackAPI) UpdateMessage(_ context.Context, req slack.UpdateMessageRequest) (slack.MessageResult, error) { + f.updates = append(f.updates, req) + if f.updateErr != nil { + return slack.MessageResult{}, f.updateErr + } + if f.updateResult != (slack.MessageResult{}) { + return f.updateResult, nil + } + return slack.MessageResult{Channel: req.Channel, TS: req.TS}, nil +} + +func (f *fakeSlackAPI) AuthTest(context.Context) error { + f.authCalls++ + return f.authErr +} + +// ---------------------------------------------------------------------- +// root_sync +// ---------------------------------------------------------------------- + +func TestSituationDelivererRootSyncPostsFirstRoot(t *testing.T) { + started := sdMustTime(t, "2026-09-05T09:00:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + deadline := now.Add(time.Minute) + contract := sdRunningTriageContract(deadline) + tr := sdTransition(1, model.LifecycleActive, contract, model.ReasonFirstAuthoritativeState, + model.JournalPublication, model.JournalData{Headline: "Situation published", OccurredAt: started}, + model.ProjectionFacts{EffectiveStartedAt: started, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, now) + summary := sdSummary(1, contract, started, now) + + fs := &fakeDelivererStore{ + episode: store.SituationEpisodeView{Summary: summary, SourceTransition: tr}, + rootOK: false, + } + api := &fakeSlackAPI{postResult: slack.MessageResult{Channel: "C-root", TS: "100.1"}} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdRootSyncIntent(tr.ID, 1, &deadline, now) + got, err := d.Deliver(context.Background(), intent) + if err != nil { + t.Fatalf("Deliver() error = %v", err) + } + if got != (NotificationDelivery{Channel: "C-root", MessageTS: "100.1", DeliveredAs: "root"}) { + t.Fatalf("Deliver() = %+v, want the posted root coordinates", got) + } + if len(api.posts) != 1 || len(api.updates) != 0 { + t.Fatalf("want exactly one PostMessage and zero UpdateMessage calls; got posts=%d updates=%d", len(api.posts), len(api.updates)) + } + if api.posts[0].Channel != "C-default" { + t.Fatalf("PostMessage channel = %q, want the configured default channel", api.posts[0].Channel) + } + if api.posts[0].ClientMsgID != intent.ClientMessageID { + t.Fatalf("PostMessage client msg id = %q, want %q", api.posts[0].ClientMsgID, intent.ClientMessageID) + } +} + +func TestSituationDelivererRootSyncUpdatesExistingRoot(t *testing.T) { + started := sdMustTime(t, "2026-09-05T09:00:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + deadline := now.Add(time.Minute) + contract := sdRunningTriageContract(deadline) + tr := sdTransition(2, model.LifecycleActive, contract, model.ReasonInvestigationStarted, + model.JournalInvestigationStarted, model.JournalData{Headline: "AlertINT investigation started", OccurredAt: started}, + model.ProjectionFacts{EffectiveStartedAt: started, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, now) + summary := sdSummary(2, contract, started, now) + + fs := &fakeDelivererStore{ + episode: store.SituationEpisodeView{Summary: summary, SourceTransition: tr}, + rootOK: true, + rootChannel: "C-existing", + rootTS: "50.5", + } + api := &fakeSlackAPI{} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdRootSyncIntent(tr.ID, 2, &deadline, now) + got, err := d.Deliver(context.Background(), intent) + if err != nil { + t.Fatalf("Deliver() error = %v", err) + } + if got.DeliveredAs != "root" || got.Channel != "C-existing" || got.MessageTS != "50.5" { + t.Fatalf("Deliver() = %+v, want the existing root's own coordinates echoed back", got) + } + if len(api.updates) != 1 || len(api.posts) != 0 { + t.Fatalf("want exactly one UpdateMessage and zero PostMessage calls; got updates=%d posts=%d", len(api.updates), len(api.posts)) + } + if api.updates[0].Channel != "C-existing" || api.updates[0].TS != "50.5" { + t.Fatalf("UpdateMessage coordinates = %+v, want the existing root's own", api.updates[0]) + } +} + +func TestSituationDelivererRootSyncRejectsStaleSummaryVersion(t *testing.T) { + started := sdMustTime(t, "2026-09-05T09:00:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + deadline := now.Add(time.Minute) + contract := sdRunningTriageContract(deadline) + tr := sdTransition(3, model.LifecycleActive, contract, model.ReasonInvestigationStarted, + model.JournalInvestigationStarted, model.JournalData{Headline: "x", OccurredAt: started}, + model.ProjectionFacts{EffectiveStartedAt: started, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, now) + summary := sdSummary(3, contract, started, now) // current version is 3 + + fs := &fakeDelivererStore{episode: store.SituationEpisodeView{Summary: summary, SourceTransition: tr}} + api := &fakeSlackAPI{} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdRootSyncIntent(tr.ID, 1, &deadline, now) // stale: names version 1 + if _, err := d.Deliver(context.Background(), intent); err == nil { + t.Fatal("Deliver() error = nil, want an error for a stale summary version") + } + if len(api.posts) != 0 || len(api.updates) != 0 { + t.Fatal("a stale-version root_sync must never reach Slack") + } +} + +// ---------------------------------------------------------------------- +// thread_append +// ---------------------------------------------------------------------- + +func TestSituationDelivererThreadAppendPostsUnderRoot(t *testing.T) { + occurred := sdMustTime(t, "2026-09-05T09:15:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + tr := sdTransition(4, model.LifecycleActive, sdRunningTriageContract(now.Add(time.Hour)), + model.ReasonInvestigationStarted, model.JournalInvestigationStarted, + model.JournalData{Headline: "AlertINT investigation started", OccurredAt: occurred}, + model.ProjectionFacts{EffectiveStartedAt: occurred, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, occurred) + + fs := &fakeDelivererStore{ + transitions: map[string]model.Transition{tr.ID: tr}, + rootOK: true, rootChannel: "C-existing", rootTS: "50.5", + } + api := &fakeSlackAPI{postResult: slack.MessageResult{Channel: "C-existing", TS: "60.6"}} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdThreadIntent(model.EffectThreadAppend, tr.ID, tr.Sequence, now) + got, err := d.Deliver(context.Background(), intent) + if err != nil { + t.Fatalf("Deliver() error = %v", err) + } + if got.DeliveredAs != "thread" { + t.Fatalf("DeliveredAs = %q, want thread", got.DeliveredAs) + } + if len(api.posts) != 1 { + t.Fatalf("want exactly one PostMessage call, got %d", len(api.posts)) + } + if api.posts[0].ThreadTS != "50.5" { + t.Fatalf("ThreadTS = %q, want the root's own ts", api.posts[0].ThreadTS) + } + if api.posts[0].ReplyBroadcast { + t.Fatal("a plain thread_append must never broadcast") + } + if !strings.Contains(api.posts[0].Text, "AlertINT investigation started") { + t.Fatalf("posted text = %q, want the Transition's own headline", api.posts[0].Text) + } +} + +func TestSituationDelivererThreadAppendRequiresExistingRoot(t *testing.T) { + occurred := sdMustTime(t, "2026-09-05T09:15:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + tr := sdTransition(5, model.LifecycleActive, sdRunningTriageContract(now.Add(time.Hour)), + model.ReasonInvestigationStarted, model.JournalInvestigationStarted, + model.JournalData{Headline: "x", OccurredAt: occurred}, + model.ProjectionFacts{EffectiveStartedAt: occurred, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, occurred) + + fs := &fakeDelivererStore{transitions: map[string]model.Transition{tr.ID: tr}, rootOK: false} + api := &fakeSlackAPI{} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdThreadIntent(model.EffectThreadAppend, tr.ID, tr.Sequence, now) + if _, err := d.Deliver(context.Background(), intent); err == nil { + t.Fatal("Deliver() error = nil, want an error: a root must be delivered before a reply is claimable") + } + if len(api.posts) != 0 { + t.Fatal("must never post a reply without an existing root") + } +} + +// ---------------------------------------------------------------------- +// broadcast_handoff +// ---------------------------------------------------------------------- + +func TestSituationDelivererBroadcastHandoffCurrentBroadcasts(t *testing.T) { + occurred := sdMustTime(t, "2026-09-05T09:15:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + action := model.OperatorActionInvestigateSituation + contract := model.ActionContract{ + NextActor: model.NextActorOperator, OperatorActionRequired: &action, + NextUpdateAt: sdTimePtr(now.Add(time.Hour)), NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnMaterialInput}, + } + tr := sdTransition(6, model.LifecycleActive, contract, model.ReasonOperatorContractChanged, + model.JournalOperatorContractChanged, model.JournalData{Headline: "Operator action required: investigate_situation", OccurredAt: occurred}, + model.ProjectionFacts{EffectiveStartedAt: occurred, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, occurred) + summary := sdSummary(6, contract, occurred, now) // SourceTransitionSequence == 6: still current + + fs := &fakeDelivererStore{ + episode: store.SituationEpisodeView{Summary: summary, SourceTransition: tr}, + transitions: map[string]model.Transition{tr.ID: tr}, + rootOK: true, rootChannel: "C-existing", rootTS: "50.5", + } + api := &fakeSlackAPI{} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdThreadIntent(model.EffectBroadcastHandoff, tr.ID, tr.Sequence, now) + got, err := d.Deliver(context.Background(), intent) + if err != nil { + t.Fatalf("Deliver() error = %v", err) + } + if got.DeliveredAs != "broadcast" { + t.Fatalf("DeliveredAs = %q, want broadcast", got.DeliveredAs) + } + if len(api.posts) != 1 || !api.posts[0].ReplyBroadcast { + t.Fatalf("want exactly one broadcast PostMessage call; got %+v", api.posts) + } + if strings.Contains(sdBlocksText(t, api.posts[0].Blocks), "no longer current") { + t.Fatal("a still-current handoff must not render as no-longer-current") + } +} + +func TestSituationDelivererBroadcastHandoffStaleDemotesToDelayedThread(t *testing.T) { + occurred := sdMustTime(t, "2026-09-05T09:15:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + action := model.OperatorActionInvestigateSituation + handoffContract := model.ActionContract{ + NextActor: model.NextActorOperator, OperatorActionRequired: &action, + NextUpdateAt: sdTimePtr(now.Add(time.Hour)), NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnMaterialInput}, + } + handoffTr := sdTransition(7, model.LifecycleActive, handoffContract, model.ReasonOperatorContractChanged, + model.JournalOperatorContractChanged, model.JournalData{Headline: "Operator action required: investigate_situation", OccurredAt: occurred}, + model.ProjectionFacts{EffectiveStartedAt: occurred, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, occurred) + + // A LATER Transition has since superseded it: the current summary's + // source sequence has moved on to 8. + laterContract := sdRunningTriageContract(now.Add(time.Hour)) + laterTr := sdTransition(8, model.LifecycleActive, laterContract, model.ReasonInvestigationStarted, + model.JournalInvestigationStarted, model.JournalData{Headline: "AlertINT investigation started", OccurredAt: now}, + model.ProjectionFacts{EffectiveStartedAt: occurred, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, now) + summary := sdSummary(8, laterContract, occurred, now) + + fs := &fakeDelivererStore{ + episode: store.SituationEpisodeView{Summary: summary, SourceTransition: laterTr}, + transitions: map[string]model.Transition{handoffTr.ID: handoffTr}, + rootOK: true, rootChannel: "C-existing", rootTS: "50.5", + } + api := &fakeSlackAPI{} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdThreadIntent(model.EffectBroadcastHandoff, handoffTr.ID, handoffTr.Sequence, now) + got, err := d.Deliver(context.Background(), intent) + if err != nil { + t.Fatalf("Deliver() error = %v", err) + } + if got.DeliveredAs != "delayed_thread" { + t.Fatalf("DeliveredAs = %q, want delayed_thread", got.DeliveredAs) + } + if len(api.posts) != 1 || api.posts[0].ReplyBroadcast { + t.Fatalf("a stale handoff must post a plain (non-broadcast) reply; got %+v", api.posts) + } + if !strings.Contains(sdBlocksText(t, api.posts[0].Blocks), "no longer current") { + t.Fatalf("posted body = %q, want a no-longer-current marker", sdBlocksText(t, api.posts[0].Blocks)) + } + // The durable ledger row itself is never mutated. + if handoffTr.Journal.Delayed || handoffTr.Journal.NoLongerCurrent { + t.Fatal("the original Transition value must never be mutated") + } +} + +// ---------------------------------------------------------------------- +// installation_gap_recovery +// ---------------------------------------------------------------------- + +func TestSituationDelivererGapRecoveryPostsSystemNotice(t *testing.T) { + opened := sdMustTime(t, "2026-09-05T09:00:00Z") + recovered := sdMustTime(t, "2026-09-05T09:10:00Z") + now := recovered + + fs := &fakeDelivererStore{gap: GapSnapshot{ + ID: "gap-1", OpenedAt: opened, RecoveredAt: recovered, AffectedSituationCount: 2, DelayedEffectCount: 5, + }} + api := &fakeSlackAPI{postResult: slack.MessageResult{Channel: "C-default", TS: "70.7"}} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdGapIntent("gap-1", now) + got, err := d.Deliver(context.Background(), intent) + if err != nil { + t.Fatalf("Deliver() error = %v", err) + } + if got != (NotificationDelivery{Channel: "C-default", MessageTS: "70.7", DeliveredAs: "system"}) { + t.Fatalf("Deliver() = %+v, want the posted system notice coordinates", got) + } + if len(api.posts) != 1 || api.posts[0].Channel != "C-default" { + t.Fatalf("want exactly one PostMessage to the default channel; got %+v", api.posts) + } + if api.posts[0].ThreadTS != "" || api.posts[0].ReplyBroadcast { + t.Fatal("the installation gap notice is a root post, never a thread reply") + } + if !strings.Contains(api.posts[0].Text, "2 Situation(s)") { + t.Fatalf("posted text = %q, want the affected-Situation count", api.posts[0].Text) + } +} + +// ---------------------------------------------------------------------- +// Probe +// ---------------------------------------------------------------------- + +func TestSituationDelivererProbeCallsAuthTest(t *testing.T) { + api := &fakeSlackAPI{} + d := NewSituationDeliverer(&fakeDelivererStore{}, api, "C-default", nil) + if err := d.Probe(context.Background()); err != nil { + t.Fatalf("Probe() error = %v", err) + } + if api.authCalls != 1 { + t.Fatalf("auth.test calls = %d, want 1", api.authCalls) + } +} + +func TestSituationDelivererProbePropagatesFailure(t *testing.T) { + wantErr := &slack.APIError{Class: slack.ErrorClassConfiguration, Code: "invalid_auth"} + api := &fakeSlackAPI{authErr: wantErr} + d := NewSituationDeliverer(&fakeDelivererStore{}, api, "C-default", nil) + err := d.Probe(context.Background()) + if !errors.Is(err, wantErr) { + var apiErr *slack.APIError + if !errors.As(err, &apiErr) || apiErr.Code != "invalid_auth" { + t.Fatalf("Probe() error = %v, want it to propagate the configuration-blocking failure", err) + } + } +} + +// ---------------------------------------------------------------------- +// recoveryEverObserved ledger scan +// ---------------------------------------------------------------------- + +func TestSituationDelivererRootSyncScansLedgerOnlyForClosedUnknownWithoutOwnRecovery(t *testing.T) { + started := sdMustTime(t, "2026-09-05T09:00:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + terminalReason := model.TerminalReasonObservationDeadline + + cases := []struct { + name string + lifecycle model.Lifecycle + ownRecoveryAt *time.Time + ledgerHasRecover bool + wantScan bool + }{ + { + name: "nonterminal: never scans", + lifecycle: model.LifecycleActive, + wantScan: false, + }, + { + name: "closed_unknown with its own recovery observation: never scans", + lifecycle: model.LifecycleClosedUnknown, + ownRecoveryAt: sdTimePtr(started.Add(time.Hour)), + ledgerHasRecover: true, // irrelevant; must not even be consulted + wantScan: false, + }, + { + name: "closed_unknown with no own recovery: scans the ledger", + lifecycle: model.LifecycleClosedUnknown, + ledgerHasRecover: true, + wantScan: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + contract := sdTerminalContract() + deadlinePtr := (*time.Time)(nil) + if c.lifecycle == model.LifecycleActive { + d := now.Add(time.Minute) + deadlinePtr = &d + contract = sdRunningTriageContract(d) + } + proj := model.ProjectionFacts{EffectiveStartedAt: started, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload} + if c.lifecycle == model.LifecycleClosedUnknown { + proj.RecoveryObservedAt = c.ownRecoveryAt + proj.TerminalAt = &now + proj.TerminalReason = &terminalReason + } + tr := sdTransition(9, c.lifecycle, contract, model.ReasonClosedUnknown, + model.JournalClosedUnknown, model.JournalData{Headline: "x", OccurredAt: now}, proj, now) + if c.lifecycle == model.LifecycleActive { + tr.Reason = model.ReasonInvestigationStarted + tr.JournalKind = model.JournalInvestigationStarted + } + summary := sdSummary(9, contract, started, now) + if c.lifecycle == model.LifecycleClosedUnknown { + summary.TerminalAt = &now + summary.FinalOutcome = "Closed with uncertainty (observation_deadline)" + } + + var ledger []model.Transition + if c.ledgerHasRecover { + recoveredAt := started.Add(30 * time.Minute) + ledger = append(ledger, sdTransition(2, model.LifecycleRecoveryPending, contract, + model.ReasonRecoveryObserved, model.JournalRecoveryPending, + model.JournalData{Headline: "Recovery observed", OccurredAt: recoveredAt}, + model.ProjectionFacts{EffectiveStartedAt: started, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload, + RecoveryObservedAt: &recoveredAt}, recoveredAt)) + } + + fs := &fakeDelivererStore{ + episode: store.SituationEpisodeView{Summary: summary, SourceTransition: tr}, + ledger: ledger, + } + api := &fakeSlackAPI{} + deliverer := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + + intent := sdRootSyncIntent(tr.ID, 9, deadlinePtr, now) + if _, err := deliverer.Deliver(context.Background(), intent); err != nil { + t.Fatalf("Deliver() error = %v", err) + } + scanned := fs.listCalls > 0 + if scanned != c.wantScan { + t.Fatalf("ledger scanned = %v, want %v (listCalls=%d)", scanned, c.wantScan, fs.listCalls) + } + }) + } +} + +// ---------------------------------------------------------------------- +// Deliver never writes Store state — a compile-time property of +// DelivererStore's own shape (read-only methods only), asserted here by +// confirming the fake's read methods are the only ones Deliver ever calls +// across every effect class exercised above (no method on +// fakeDelivererStore records a "write"; if Deliver ever needed one, this +// interface satisfaction would not compile). +func TestSituationDelivererNeverWritesStoreState(t *testing.T) { + var _ DelivererStore = (*fakeDelivererStore)(nil) +} + +// ---------------------------------------------------------------------- +// Deliver validates the intent up front. +// ---------------------------------------------------------------------- + +func TestSituationDelivererDeliverRejectsInvalidIntent(t *testing.T) { + api := &fakeSlackAPI{} + d := NewSituationDeliverer(&fakeDelivererStore{}, api, "C-default", nil) + if _, err := d.Deliver(context.Background(), model.NotificationIntent{}); err == nil { + t.Fatal("Deliver() error = nil, want an error for an invalid intent") + } + if len(api.posts) != 0 || len(api.updates) != 0 { + t.Fatal("an invalid intent must never reach Slack") + } +} diff --git a/internal/notify/slack/api_client.go b/internal/notify/slack/api_client.go new file mode 100644 index 0000000..6ae4063 --- /dev/null +++ b/internal/notify/slack/api_client.go @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package slack + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + slacklib "github.com/slack-go/slack" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 6: a narrow, hand-rolled Slack Web API client covering +// exactly chat.postMessage, chat.update, and auth.test — the three calls +// Situation delivery needs. It follows this codebase's existing narrow- +// client convention (internal/zabbix/client.go, internal/prometheus/ +// client.go) rather than wrapping slack-go/slack's own HTTP client: this +// file needs precise, independently testable control over classifying a +// response as retryable, configuration-blocking, or invalid (spec.md +// "Notification intent contract"), which slack-go's client does not +// expose as typed errors. Block Kit body construction still reuses +// slack-go's own block/text-object structs (situation.go) — they are +// plain, well-tested wire-compatible data types, not the HTTP surface +// this file replaces. +// +// This client requests no Slack history-read scopes, never searches +// Slack, and performs no read-before-redrive reconciliation (spec.md +// "Local idempotency, external delivery, and ordering"). +// ---------------------------------------------------------------------- + +const ( + defaultAPIBase = "https://slack.com/api" + defaultTimeout = 10 * time.Second + maxResponseBodyBytes = 1 << 20 // 1 MiB: generous for any chat.postMessage/update/auth.test reply, bounded against a misbehaving peer. +) + +// ErrorClass is the closed classification api_client.go assigns to every +// failed Slack call (Task 6 brief Step 4/5): transport/5xx/rate-limit/ +// uncertain responses are retryable; token/scope/channel rejections are +// configuration-blocking; a malformed durable payload this build sent is +// invalid. +type ErrorClass string + +const ( + ErrorClassRetryable ErrorClass = "retryable" + ErrorClassConfiguration ErrorClass = "configuration_blocking" + ErrorClassInvalid ErrorClass = "invalid" +) + +// APIError is the typed result of a failed Slack Web API call. It never +// carries the bot token or a raw response body — only a bounded class, +// a bounded Slack (or local transport) error code, and, for a retryable +// failure, how long to wait before trying again. +type APIError struct { + Class ErrorClass + Code string + RetryAfter time.Duration +} + +func (e *APIError) Error() string { + if e.RetryAfter > 0 { + return fmt.Sprintf("slack: %s: %s (retry after %s)", e.Class, e.Code, e.RetryAfter) + } + return fmt.Sprintf("slack: %s: %s", e.Class, e.Code) +} + +// Config configures the narrow Slack Web API Client. +type Config struct { + BotToken string + BaseURL string // override for tests; defaults to defaultAPIBase. + TimeoutSeconds int +} + +// Client is a narrow Slack Web API client covering chat.postMessage, +// chat.update, and auth.test. +type Client struct { + httpClient *http.Client + token string + baseURL string +} + +// NewClient constructs a Client from cfg. +func NewClient(cfg Config) *Client { + timeout := time.Duration(cfg.TimeoutSeconds) * time.Second + if timeout <= 0 { + timeout = defaultTimeout + } + base := cfg.BaseURL + if base == "" { + base = defaultAPIBase + } + return &Client{ + httpClient: &http.Client{Timeout: timeout}, + token: cfg.BotToken, + baseURL: strings.TrimRight(base, "/"), + } +} + +// PostMessageRequest is one chat.postMessage call. +type PostMessageRequest struct { + Channel string + Text string + Blocks []slacklib.Block + ThreadTS string // set for a thread reply; empty for a root. + + // ReplyBroadcast marks a thread reply as also shown in the channel + // (Slack's reply_broadcast). Root posts must leave this false. + ReplyBroadcast bool + + // ClientMsgID is this build's own deterministic local identity for one + // durable delivery (Task 4's NotificationIntent.ClientMessageID). Slack's + // chat.postMessage accepts no idempotency parameter (spec.md: "Plan 3 + // requests no Slack history-read scopes and performs no + // read-before-redrive reconciliation" — an uncertain retry may rarely + // create a provider-side duplicate, accepted). It is carried only as a + // request header for local log/trace correlation, never sent as a body + // field Slack would interpret, and never logged or returned inside an + // error. + ClientMsgID string +} + +// UpdateMessageRequest is one chat.update call. +type UpdateMessageRequest struct { + Channel string + TS string + Text string + Blocks []slacklib.Block + ClientMsgID string +} + +// MessageResult is the durable Slack coordinate a successful post/update +// returns. +type MessageResult struct { + Channel string + TS string +} + +// clientMsgIDHeader carries PostMessageRequest/UpdateMessageRequest's own +// ClientMsgID for request/response log correlation only. Slack does not +// interpret it. +const clientMsgIDHeader = "X-Alertint-Client-Message-Id" + +// PostMessage posts one new message (a root, a thread reply, or a +// broadcast reply per ThreadTS/ReplyBroadcast). +func (c *Client) PostMessage(ctx context.Context, req PostMessageRequest) (MessageResult, error) { + if err := validatePostMessageRequest(req); err != nil { + return MessageResult{}, err + } + payload := postMessagePayload{ + Channel: req.Channel, + Text: req.Text, + Blocks: req.Blocks, + ThreadTS: req.ThreadTS, + ReplyBroadcast: req.ReplyBroadcast, + } + env, err := c.call(ctx, "chat.postMessage", payload, req.ClientMsgID) + if err != nil { + return MessageResult{}, err + } + return MessageResult{Channel: env.Channel, TS: env.TS}, nil +} + +// UpdateMessage edits a previously posted message in place (including the +// R4 deadline-refresh root edit, which is a plain chat.update). +func (c *Client) UpdateMessage(ctx context.Context, req UpdateMessageRequest) (MessageResult, error) { + if err := validateUpdateMessageRequest(req); err != nil { + return MessageResult{}, err + } + payload := updateMessagePayload{ + Channel: req.Channel, + TS: req.TS, + Text: req.Text, + Blocks: req.Blocks, + } + env, err := c.call(ctx, "chat.update", payload, req.ClientMsgID) + if err != nil { + return MessageResult{}, err + } + return MessageResult{Channel: env.Channel, TS: env.TS}, nil +} + +// AuthTest verifies the configured bot token against auth.test — the +// startup/recovery readiness probe. +func (c *Client) AuthTest(ctx context.Context) error { + _, err := c.call(ctx, "auth.test", struct{}{}, "") + return err +} + +// ---------------------------------------------------------------------- +// Validation — malformed durable payloads this build would otherwise send. +// ---------------------------------------------------------------------- + +func validatePostMessageRequest(req PostMessageRequest) error { + if strings.TrimSpace(req.Channel) == "" { + return &APIError{Class: ErrorClassInvalid, Code: "missing_channel"} + } + if strings.TrimSpace(req.Text) == "" && len(req.Blocks) == 0 { + return &APIError{Class: ErrorClassInvalid, Code: "missing_text"} + } + if req.ReplyBroadcast && strings.TrimSpace(req.ThreadTS) == "" { + return &APIError{Class: ErrorClassInvalid, Code: "broadcast_without_thread"} + } + return nil +} + +func validateUpdateMessageRequest(req UpdateMessageRequest) error { + if strings.TrimSpace(req.Channel) == "" { + return &APIError{Class: ErrorClassInvalid, Code: "missing_channel"} + } + if strings.TrimSpace(req.TS) == "" { + return &APIError{Class: ErrorClassInvalid, Code: "missing_ts"} + } + if strings.TrimSpace(req.Text) == "" && len(req.Blocks) == 0 { + return &APIError{Class: ErrorClassInvalid, Code: "missing_text"} + } + return nil +} + +// ---------------------------------------------------------------------- +// Transport +// ---------------------------------------------------------------------- + +type postMessagePayload struct { + Channel string `json:"channel"` + Text string `json:"text"` + Blocks []slacklib.Block `json:"blocks,omitempty"` + ThreadTS string `json:"thread_ts,omitempty"` + ReplyBroadcast bool `json:"reply_broadcast,omitempty"` +} + +type updateMessagePayload struct { + Channel string `json:"channel"` + TS string `json:"ts"` + Text string `json:"text"` + Blocks []slacklib.Block `json:"blocks,omitempty"` +} + +// apiEnvelope is the common Slack Web API response shape this client +// needs: success/error plus the delivered coordinates chat.postMessage/ +// chat.update return. auth.test's own extra fields (team, user, ...) are +// not read — a non-error response is success enough for a readiness +// probe. +type apiEnvelope struct { + OK bool `json:"ok"` + Error string `json:"error,omitempty"` + Channel string `json:"channel,omitempty"` + TS string `json:"ts,omitempty"` +} + +// call issues one Slack Web API method with a JSON body and classifies +// every failure mode: a transport error, a non-2xx HTTP status, an +// undecodable body, or a Slack-reported {"ok":false,"error":"..."} — never +// returning the raw response body or the bot token in the resulting error. +func (c *Client) call(ctx context.Context, method string, payload any, clientMsgID string) (apiEnvelope, error) { + body, err := json.Marshal(payload) + if err != nil { + return apiEnvelope{}, &APIError{Class: ErrorClassInvalid, Code: "encode_request_failed"} + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/"+method, bytes.NewReader(body)) + if err != nil { + return apiEnvelope{}, &APIError{Class: ErrorClassInvalid, Code: "build_request_failed"} + } + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Authorization", "Bearer "+c.token) + if clientMsgID != "" { + req.Header.Set(clientMsgIDHeader, clientMsgID) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return apiEnvelope{}, classifyTransportError(err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBodyBytes)) + if err != nil { + return apiEnvelope{}, &APIError{Class: ErrorClassRetryable, Code: "read_response_failed"} + } + + switch { + case resp.StatusCode == http.StatusTooManyRequests: + return apiEnvelope{}, &APIError{Class: ErrorClassRetryable, Code: "ratelimited", RetryAfter: retryAfterFrom(resp.Header)} + case resp.StatusCode >= 500: + return apiEnvelope{}, &APIError{Class: ErrorClassRetryable, Code: fmt.Sprintf("http_%d", resp.StatusCode)} + case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: + return apiEnvelope{}, &APIError{Class: ErrorClassConfiguration, Code: fmt.Sprintf("http_%d", resp.StatusCode)} + case resp.StatusCode >= 400: + // An unexpected 4xx this build does not specifically classify below + // (Slack's Web API normally answers 200 with ok:false instead): the + // request itself may be malformed, but the outcome is not certain + // enough to call configuration-blocking — treat it as invalid so + // it surfaces rather than retries forever. + return apiEnvelope{}, &APIError{Class: ErrorClassInvalid, Code: fmt.Sprintf("http_%d", resp.StatusCode)} + } + + var envelope apiEnvelope + if err := json.Unmarshal(respBody, &envelope); err != nil { + // Slack answered 2xx but not with a shape this client understands: + // the outcome is uncertain, so treat it as retryable rather than + // assume either success or failure. + return apiEnvelope{}, &APIError{Class: ErrorClassRetryable, Code: "undecodable_response"} + } + if !envelope.OK { + return apiEnvelope{}, classifySlackErrorCode(envelope.Error, resp.Header) + } + return envelope, nil +} + +// classifyTransportError classifies a failure to even complete the HTTP +// round trip (DNS, connection refused, TLS, timeout, context +// cancellation) as retryable — the outcome of such a failure is always +// uncertain, never a confirmed rejection. +func classifyTransportError(err error) *APIError { + code := "transport_error" + if errors.Is(err, context.DeadlineExceeded) || isTimeoutErr(err) { + code = "timeout" + } + return &APIError{Class: ErrorClassRetryable, Code: code} +} + +func isTimeoutErr(err error) bool { + var timeoutErr interface{ Timeout() bool } + return errors.As(err, &timeoutErr) && timeoutErr.Timeout() +} + +// configurationErrorCodes are Slack Web API error codes this build treats +// as blocked_configuration: missing/invalid token, scope, channel, +// authentication, or permission configuration (spec.md "Required fields +// and states": "blocked_configuration covers missing/invalid token, +// channel, authentication, or permission configuration"). +var configurationErrorCodes = map[string]bool{ + "invalid_auth": true, + "not_authed": true, + "account_inactive": true, + "token_revoked": true, + "token_expired": true, + "no_permission": true, + "missing_scope": true, + "channel_not_found": true, + "not_in_channel": true, + "is_archived": true, + "restricted_action": true, + "restricted_action_non_threadable_channel": true, + "restricted_action_read_only_channel": true, + "restricted_action_thread_only_channel": true, + "ekm_access_denied": true, + "org_login_required": true, + "not_allowed_token_type": true, + "method_not_supported_for_channel_type": true, + "team_access_not_granted": true, + "user_is_bot": true, + "user_is_restricted": true, +} + +// retryableErrorCodes are Slack Web API error codes this build treats as +// retryable: rate limiting and Slack-side transient failures. +var retryableErrorCodes = map[string]bool{ + "ratelimited": true, + "rate_limited": true, + "internal_error": true, + "fatal_error": true, + "request_timeout": true, + "service_unavailable": true, +} + +// classifySlackErrorCode classifies one {"ok":false,"error":code} response. +// Any code this build does not specifically recognize as retryable or +// configuration-blocking is invalid — a malformed durable payload this +// build sent (e.g. invalid_blocks, msg_too_long, message_not_found), which +// stays operator-visible and redriveable rather than retried forever. +func classifySlackErrorCode(code string, header http.Header) *APIError { + switch { + case retryableErrorCodes[code]: + return &APIError{Class: ErrorClassRetryable, Code: code, RetryAfter: retryAfterFrom(header)} + case configurationErrorCodes[code]: + return &APIError{Class: ErrorClassConfiguration, Code: code} + case code == "": + return &APIError{Class: ErrorClassInvalid, Code: "unknown_error"} + default: + return &APIError{Class: ErrorClassInvalid, Code: code} + } +} + +// retryAfterFrom parses the Retry-After header (seconds, per Slack's own +// rate-limit documentation) into a Duration. Zero when absent or +// unparsable — the caller applies its own backoff floor in that case. +func retryAfterFrom(header http.Header) time.Duration { + v := header.Get("Retry-After") + if v == "" { + return 0 + } + seconds, err := strconv.Atoi(strings.TrimSpace(v)) + if err != nil || seconds < 0 { + return 0 + } + return time.Duration(seconds) * time.Second +} diff --git a/internal/notify/slack/api_client_test.go b/internal/notify/slack/api_client_test.go new file mode 100644 index 0000000..4d9a5fd --- /dev/null +++ b/internal/notify/slack/api_client_test.go @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package slack + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + slacklib "github.com/slack-go/slack" +) + +func testClient(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return NewClient(Config{BotToken: "xoxb-test-token", BaseURL: srv.URL, TimeoutSeconds: 2}) +} + +func decodeBody(t *testing.T, r *http.Request) map[string]any { + t.Helper() + var m map[string]any + if err := json.NewDecoder(r.Body).Decode(&m); err != nil { + t.Fatalf("decode request body: %v", err) + } + return m +} + +// ---------------------------------------------------------------------- +// chat.postMessage +// ---------------------------------------------------------------------- + +func TestSlackAPIPostMessageSuccess(t *testing.T) { + var gotPath string + var gotAuth string + var gotBody map[string]any + var gotClientMsgID string + + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + gotClientMsgID = r.Header.Get(clientMsgIDHeader) + gotBody = decodeBody(t, r) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"channel":"C123","ts":"1700000000.000100"}`)) + }) + + res, err := c.PostMessage(context.Background(), PostMessageRequest{ + Channel: "C123", + Text: "hello", + Blocks: []slacklib.Block{slacklib.NewSectionBlock(slacklib.NewTextBlockObject(slacklib.MarkdownType, "hello", false, false), nil, nil)}, + ClientMsgID: "client-msg-1", + }) + if err != nil { + t.Fatalf("PostMessage() error = %v", err) + } + if res.Channel != "C123" || res.TS != "1700000000.000100" { + t.Fatalf("PostMessage() = %+v, want channel C123 ts 1700000000.000100", res) + } + if !strings.HasSuffix(gotPath, "/chat.postMessage") { + t.Fatalf("request path = %q, want chat.postMessage", gotPath) + } + if gotAuth != "Bearer xoxb-test-token" { + t.Fatalf("Authorization header = %q, want Bearer xoxb-test-token", gotAuth) + } + if gotClientMsgID != "client-msg-1" { + t.Fatalf("client message id header = %q, want client-msg-1", gotClientMsgID) + } + if gotBody["channel"] != "C123" || gotBody["text"] != "hello" { + t.Fatalf("request body = %+v, want channel/text set", gotBody) + } + if _, ok := gotBody["thread_ts"]; ok { + t.Fatalf("request body = %+v, a root post must not set thread_ts", gotBody) + } + if _, ok := gotBody["reply_broadcast"]; ok { + t.Fatalf("request body = %+v, a root post must not set reply_broadcast", gotBody) + } + blocks, ok := gotBody["blocks"].([]any) + if !ok || len(blocks) != 1 { + t.Fatalf("request body blocks = %+v, want exactly one block", gotBody["blocks"]) + } +} + +func TestSlackAPIPostMessageStableClientMsgID(t *testing.T) { + // Two calls with identical input must send byte-identical request + // bodies and the same client-message-id header — no randomness or + // timestamps leak into the wire request. + var bodies []string + var headers []string + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + body := decodeBody(t, r) + encoded, _ := json.Marshal(body) + bodies = append(bodies, string(encoded)) + headers = append(headers, r.Header.Get(clientMsgIDHeader)) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"channel":"C123","ts":"1.1"}`)) + }) + + req := PostMessageRequest{Channel: "C123", Text: "hello", ClientMsgID: "stable-id"} + for i := 0; i < 2; i++ { + if _, err := c.PostMessage(context.Background(), req); err != nil { + t.Fatalf("PostMessage() [%d] error = %v", i, err) + } + } + if bodies[0] != bodies[1] { + t.Fatalf("request bodies differ across identical calls: %q vs %q", bodies[0], bodies[1]) + } + if headers[0] != "stable-id" || headers[1] != "stable-id" { + t.Fatalf("client message id headers = %v, want [stable-id stable-id]", headers) + } +} + +func TestSlackAPIPostMessageThreadAndBroadcastFields(t *testing.T) { + cases := []struct { + name string + req func(base PostMessageRequest) PostMessageRequest + wantThreadTS string + wantBroadcast bool + }{ + { + name: "root", + req: func(b PostMessageRequest) PostMessageRequest { return b }, + wantThreadTS: "", + }, + { + name: "thread reply", + req: func(b PostMessageRequest) PostMessageRequest { + b.ThreadTS = "1700000000.000100" + return b + }, + wantThreadTS: "1700000000.000100", + }, + { + name: "broadcast reply", + req: func(b PostMessageRequest) PostMessageRequest { + b.ThreadTS = "1700000000.000100" + b.ReplyBroadcast = true + return b + }, + wantThreadTS: "1700000000.000100", + wantBroadcast: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var gotBody map[string]any + client := testClient(t, func(w http.ResponseWriter, r *http.Request) { + gotBody = decodeBody(t, r) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"channel":"C123","ts":"2.2"}`)) + }) + req := c.req(PostMessageRequest{Channel: "C123", Text: "hi"}) + if _, err := client.PostMessage(context.Background(), req); err != nil { + t.Fatalf("PostMessage() error = %v", err) + } + if ts, _ := gotBody["thread_ts"].(string); ts != c.wantThreadTS { + t.Fatalf("thread_ts = %q, want %q", ts, c.wantThreadTS) + } + broadcast, _ := gotBody["reply_broadcast"].(bool) + if broadcast != c.wantBroadcast { + t.Fatalf("reply_broadcast = %v, want %v", broadcast, c.wantBroadcast) + } + }) + } +} + +// ---------------------------------------------------------------------- +// chat.update +// ---------------------------------------------------------------------- + +func TestSlackAPIUpdateMessageSuccess(t *testing.T) { + var gotBody map[string]any + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/chat.update") { + t.Fatalf("request path = %q, want chat.update", r.URL.Path) + } + gotBody = decodeBody(t, r) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"channel":"C123","ts":"1700000000.000100"}`)) + }) + + res, err := c.UpdateMessage(context.Background(), UpdateMessageRequest{ + Channel: "C123", TS: "1700000000.000100", Text: "updated", + }) + if err != nil { + t.Fatalf("UpdateMessage() error = %v", err) + } + if res.Channel != "C123" || res.TS != "1700000000.000100" { + t.Fatalf("UpdateMessage() = %+v, want the edited coordinates echoed back", res) + } + if gotBody["ts"] != "1700000000.000100" { + t.Fatalf("request body ts = %v, want 1700000000.000100", gotBody["ts"]) + } +} + +// ---------------------------------------------------------------------- +// auth.test +// ---------------------------------------------------------------------- + +func TestSlackAPIAuthTestSuccess(t *testing.T) { + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/auth.test") { + t.Fatalf("request path = %q, want auth.test", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"team":"T1","user":"alertint"}`)) + }) + if err := c.AuthTest(context.Background()); err != nil { + t.Fatalf("AuthTest() error = %v", err) + } +} + +func TestSlackAPIAuthTestInvalidAuthIsConfigurationBlocking(t *testing.T) { + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"invalid_auth"}`)) + }) + err := c.AuthTest(context.Background()) + assertAPIError(t, err, ErrorClassConfiguration, "invalid_auth") +} + +// ---------------------------------------------------------------------- +// Classification: retryable / configuration-blocking / invalid. +// ---------------------------------------------------------------------- + +func assertAPIError(t *testing.T, err error, wantClass ErrorClass, wantCode string) { + t.Helper() + if err == nil { + t.Fatal("error = nil, want an APIError") + } + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error = %v (%T), want *APIError", err, err) + } + if apiErr.Class != wantClass { + t.Fatalf("error class = %q, want %q (err=%v)", apiErr.Class, wantClass, err) + } + if wantCode != "" && apiErr.Code != wantCode { + t.Fatalf("error code = %q, want %q", apiErr.Code, wantCode) + } +} + +func TestSlackAPIClassifiesRetryable(t *testing.T) { + cases := []struct { + name string + handler http.HandlerFunc + wantCode string + wantRetry time.Duration + }{ + { + name: "http 500", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }, + wantCode: "http_500", + }, + { + name: "http 503", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }, + wantCode: "http_503", + }, + { + name: "http 429 with Retry-After", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "42") + w.WriteHeader(http.StatusTooManyRequests) + }, + wantCode: "ratelimited", + wantRetry: 42 * time.Second, + }, + { + name: "slack ratelimited error body", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "5") + _, _ = w.Write([]byte(`{"ok":false,"error":"ratelimited"}`)) + }, + wantCode: "ratelimited", + wantRetry: 5 * time.Second, + }, + { + name: "slack internal_error", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"internal_error"}`)) + }, + wantCode: "internal_error", + }, + { + name: "undecodable 200 body", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`not json`)) + }, + wantCode: "undecodable_response", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := testClient(t, c.handler) + err := client.AuthTest(context.Background()) + assertAPIError(t, err, ErrorClassRetryable, c.wantCode) + var apiErr *APIError + errors.As(err, &apiErr) + if apiErr.RetryAfter != c.wantRetry { + t.Fatalf("RetryAfter = %s, want %s", apiErr.RetryAfter, c.wantRetry) + } + }) + } +} + +func TestSlackAPIClassifiesTransportFailureAsRetryable(t *testing.T) { + // A closed server: the request never gets a response at all. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + url := srv.URL + srv.Close() + + c := NewClient(Config{BotToken: "xoxb-test", BaseURL: url, TimeoutSeconds: 1}) + err := c.AuthTest(context.Background()) + assertAPIError(t, err, ErrorClassRetryable, "transport_error") +} + +func TestSlackAPIClassifiesTimeoutAsRetryable(t *testing.T) { + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(500 * time.Millisecond): + } + }) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err := c.AuthTest(ctx) + assertAPIError(t, err, ErrorClassRetryable, "timeout") +} + +func TestSlackAPIClassifiesConfigurationBlocking(t *testing.T) { + cases := []struct { + name string + handler http.HandlerFunc + code string + }{ + { + name: "invalid_auth", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"invalid_auth"}`)) + }, + code: "invalid_auth", + }, + { + name: "missing_scope", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"missing_scope"}`)) + }, + code: "missing_scope", + }, + { + name: "channel_not_found", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"channel_not_found"}`)) + }, + code: "channel_not_found", + }, + { + name: "not_in_channel", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"not_in_channel"}`)) + }, + code: "not_in_channel", + }, + { + name: "http 401", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }, + code: "http_401", + }, + { + name: "http 403", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + }, + code: "http_403", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := testClient(t, c.handler) + err := client.AuthTest(context.Background()) + assertAPIError(t, err, ErrorClassConfiguration, c.code) + }) + } +} + +func TestSlackAPIClassifiesInvalidPayload(t *testing.T) { + cases := []struct { + name string + handler http.HandlerFunc + code string + }{ + { + name: "invalid_blocks", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"invalid_blocks"}`)) + }, + code: "invalid_blocks", + }, + { + name: "msg_too_long", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"msg_too_long"}`)) + }, + code: "msg_too_long", + }, + { + name: "message_not_found", + handler: func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":false,"error":"message_not_found"}`)) + }, + code: "message_not_found", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := testClient(t, c.handler) + err := client.AuthTest(context.Background()) + assertAPIError(t, err, ErrorClassInvalid, c.code) + }) + } +} + +func TestSlackAPIPostMessageValidatesLocallyBeforeCalling(t *testing.T) { + called := false + c := testClient(t, func(w http.ResponseWriter, r *http.Request) { + called = true + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"channel":"C","ts":"1"}`)) + }) + + cases := []struct { + name string + req PostMessageRequest + }{ + {"missing channel", PostMessageRequest{Text: "hi"}}, + {"missing text and blocks", PostMessageRequest{Channel: "C123"}}, + {"broadcast without thread", PostMessageRequest{Channel: "C123", Text: "hi", ReplyBroadcast: true}}, + } + for _, c2 := range cases { + t.Run(c2.name, func(t *testing.T) { + called = false + _, err := c.PostMessage(context.Background(), c2.req) + assertAPIError(t, err, ErrorClassInvalid, "") + if called { + t.Fatal("a locally-invalid request must never reach the Slack API") + } + }) + } +} + +// ---------------------------------------------------------------------- +// Redaction: never the token, never the raw response body. +// ---------------------------------------------------------------------- + +func TestSlackAPIErrorNeverLeaksTokenOrBody(t *testing.T) { + const secretToken = "xoxb-super-secret-do-not-leak" + const sensitiveBody = `{"ok":false,"error":"invalid_auth","extra_sensitive_field":"do-not-leak-this-either"}` + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(sensitiveBody)) + })) + defer srv.Close() + + c := NewClient(Config{BotToken: secretToken, BaseURL: srv.URL, TimeoutSeconds: 2}) + err := c.AuthTest(context.Background()) + if err == nil { + t.Fatal("want an error") + } + msg := err.Error() + if strings.Contains(msg, secretToken) { + t.Fatalf("error message leaked the bot token: %q", msg) + } + if strings.Contains(msg, "extra_sensitive_field") || strings.Contains(msg, "do-not-leak-this-either") { + t.Fatalf("error message leaked the raw response body: %q", msg) + } +} diff --git a/internal/notify/slack/situation.go b/internal/notify/slack/situation.go new file mode 100644 index 0000000..afc7156 --- /dev/null +++ b/internal/notify/slack/situation.go @@ -0,0 +1,675 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package slack + +// situation.go is Plan 3's pure Situation-owned Slack renderer: roots, +// immutable journal entries, and the one installation Delivery-gap +// recovery notice. It makes no publication decision, no Slack call, and no +// Store read — every function here is total over its arguments and reads +// nothing else (Task 6 brief Step 3: "Render a root only from the selected +// Episode-summary version, a journal only from its referenced Transition, +// and a recovery notice only from its gap generation"). + +import ( + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + slacklib "github.com/slack-go/slack" + + "github.com/alertint/alertint-agent/internal/situation" + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// Bounded rendering limits. maxSectionChars mirrors Slack's own Block Kit +// text-object limit (block_object.go's TextBlockObject.Validate: "text +// cannot be longer than 3000 characters") so a render never produces a +// payload Slack itself would reject. maxRenderedListEntries and +// truncationMarker back this file's own compactness choice — a card stays +// scannable — never a silent drop (Step 3: "Apply bounded truncation with +// a visible marker"). +const ( + maxSectionChars = 3000 + maxRenderedListEntries = 5 + truncationMarker = "… [truncated]" +) + +// RenderedMessage is one fully rendered Slack payload: the plain-text +// fallback Slack requires (and a client without Block Kit support shows — +// a mobile push notification, a screen reader) plus the Block Kit body. +// Blocks is nil for the one plain-text-only surface (the installation gap +// notice mirrors PostSystemMessage's own plain-text convention). +type RenderedMessage struct { + Text string + Blocks []slacklib.Block +} + +// ---------------------------------------------------------------------- +// Root +// ---------------------------------------------------------------------- + +// SituationRootInput is everything RenderSituationRoot needs: the selected, +// coherent Episode-summary/source-Transition pair (Task 5's +// GetSituationEpisodeView reads exactly this pair in one snapshot), the +// delivering root_sync intent's own committed contract deadline (R4 — +// rendered here, never read from the Episode summary), and the render-time +// instant used for the ceil-rounded countdown / overdue comparison. +// +// RecoveryEverObserved answers a question the source Transition alone +// cannot always answer: whether a recovery_pending Transition occurred +// anywhere earlier in this Situation's Transition ledger. It changes the +// rendered chain only when SourceTransition is itself a closed_unknown +// Transition reached directly from `active` with no recovery observation +// of its OWN (Projection.RecoveryObservedAt nil) — a refire clears that +// field on every Transition after it (internal/situation/controller.go's +// resolveLifecycle, EventRefired branch), so a closed_unknown that follows +// a refire cannot see the earlier recovery_pending phase through the +// source Transition alone. DeriveOrientation's own doc comment names this +// gap: "whether a closed_unknown chain includes Monitoring at all... +// additionally needs the Situation's Transition history, which the Slack +// renderer reads." Every other orientation decides the chain from +// SourceTransition alone, so a caller may safely pass false when it has +// not needed to scan the ledger (e.g. SourceTransition.Projection. +// RecoveryObservedAt is already non-nil, or the Transition is not a +// closed_unknown at all). +type SituationRootInput struct { + Summary model.EpisodeSummary + SourceTransition model.Transition + ContractDeadlineAt *time.Time + Now time.Time + RecoveryEverObserved bool +} + +// RenderSituationRoot renders one Situation's root card from exactly the +// selected Episode-summary version and its source Transition — never a +// second, newer read of either. +func RenderSituationRoot(in SituationRootInput) (RenderedMessage, error) { + if err := validateRootInput(in); err != nil { + return RenderedMessage{}, err + } + + orientation := situation.DeriveOrientation(in.Summary, in.SourceTransition) + recoveryEverObserved := in.RecoveryEverObserved || in.SourceTransition.Projection.RecoveryObservedAt != nil + terminal := in.SourceTransition.Lifecycle.Terminal() + + blocks := []slacklib.Block{ + headerBlock(in.Summary, in.SourceTransition.Drill), + orientationBlock(orientation, recoveryEverObserved), + } + + if terminal { + blocks = append(blocks, terminalBodyBlocks(in.Summary, in.SourceTransition)...) + } else { + // "The line immediately below [the orientation] renders the + // Operator contract in compact prose" (spec.md "Root and journal + // rendering") — the contract/deadline line sits directly under the + // orientation chain, before the other nonterminal sections. + contractAndDeadline, err := contractAndDeadlineBlock(orientation, in.Summary.ActionContract, in.ContractDeadlineAt, in.Now) + if err != nil { + return RenderedMessage{}, err + } + blocks = append(blocks, contractAndDeadline) + blocks = append(blocks, nonterminalBodyBlocks(in.Summary, in.SourceTransition)...) + } + blocks = append(blocks, handleBlock(in.Summary)) + + return RenderedMessage{ + Text: rootFallback(in.Summary, orientation, in.SourceTransition.Drill), + Blocks: blocks, + }, nil +} + +func validateRootInput(in SituationRootInput) error { + if err := in.Summary.Validate(); err != nil { + return fmt.Errorf("slack: render situation root: %w", err) + } + if err := in.SourceTransition.Validate(); err != nil { + return fmt.Errorf("slack: render situation root: %w", err) + } + if in.Summary.SituationID != in.SourceTransition.SituationID { + return fmt.Errorf("slack: render situation root: summary situation %q does not match transition situation %q", + in.Summary.SituationID, in.SourceTransition.SituationID) + } + if in.Summary.SourceTransitionSequence != in.SourceTransition.Sequence { + return fmt.Errorf("slack: render situation root: summary source sequence %d does not match transition sequence %d", + in.Summary.SourceTransitionSequence, in.SourceTransition.Sequence) + } + if in.Now.IsZero() { + return errors.New("slack: render situation root: now is required") + } + terminal := in.SourceTransition.Lifecycle.Terminal() + switch { + case terminal && in.ContractDeadlineAt != nil: + return errors.New("slack: render situation root: a terminal root must not carry a contract deadline") + case !terminal && in.ContractDeadlineAt == nil: + return errors.New("slack: render situation root: a nonterminal root requires the delivering intent's contract deadline (R4)") + } + return nil +} + +func headerBlock(s model.EpisodeSummary, drill bool) slacklib.Block { + return sectionBlock(drillPrefix(drill) + "*" + s.Title + "*") +} + +func orientationBlock(o situation.Orientation, recoveryEverObserved bool) slacklib.Block { + return contextBlock(renderOrientationChain(o, recoveryEverObserved)) +} + +// renderOrientationChain renders the compact phase chain with exactly one +// current phase emphasized (spec.md "Root and journal rendering"). The +// nonterminal chains always preview the full roadmap — Monitoring and the +// generic "Outcome" placeholder included — since an active Situation has +// not yet learned whether it will ever be observed recovering. A terminal +// chain instead reports what actually happened: Recovered always implies +// Monitoring (AdvanceLifecycle only reaches `recovered` from +// `recovery_pending`); closed_unknown includes Monitoring only when +// recoveryEverObserved says the episode passed through it. +func renderOrientationChain(o situation.Orientation, recoveryEverObserved bool) string { + const ( + phaseObserved = "Observed" + phaseInvestigating = "Investigating" + phaseMonitoring = "Monitoring" + phasePlaceholder = "Outcome" + ) + + outcome := phasePlaceholder + var bold string + showMonitoring := true + + switch o { + case situation.OrientationObserved: + bold = phaseObserved + case situation.OrientationInvestigating: + bold = phaseInvestigating + case situation.OrientationMonitoring: + bold = phaseMonitoring + case situation.OrientationRecovered: + outcome = "Recovered" + bold = outcome + case situation.OrientationClosedUncertain: + outcome = "Closed uncertain" + bold = outcome + showMonitoring = recoveryEverObserved + default: + bold = phaseObserved + } + + phases := []string{phaseObserved, phaseInvestigating} + if showMonitoring { + phases = append(phases, phaseMonitoring) + } + phases = append(phases, outcome) + + rendered := make([]string, len(phases)) + for i, p := range phases { + if p == bold { + rendered[i] = "*" + p + "*" + } else { + rendered[i] = p + } + } + return strings.Join(rendered, " → ") +} + +// contractAndDeadlineBlock renders who acts next and what happens next and +// by when, as one compact line, e.g. "AlertINT is running Acute Triage · +// update by 10:00:15" — the delivering intent's own ContractDeadlineAt +// (R4), never the Episode summary. +func contractAndDeadlineBlock(o situation.Orientation, c model.ActionContract, deadline *time.Time, now time.Time) (slacklib.Block, error) { + if deadline == nil { + return nil, errors.New("slack: render situation root: nonterminal root requires a contract deadline (R4)") + } + return contextBlock(contractLine(o, c) + " · " + RenderDeadline(*deadline, now)), nil +} + +// nonterminalBodyBlocks renders the still-open root's remaining required +// sections: what is happening, why current Attention is warranted, and +// what AlertINT checked or is checking. +func nonterminalBodyBlocks(s model.EpisodeSummary, t model.Transition) []slacklib.Block { + var blocks []slacklib.Block + if line := whatIsHappeningLine(s); line != "" { + blocks = append(blocks, sectionBlock(line)) + } + if line := attentionLine(s, t); line != "" { + blocks = append(blocks, sectionBlock(line)) + } + if line := investigationLine(s); line != "" { + blocks = append(blocks, sectionBlock(line)) + } + if line := recurrenceLine(s); line != "" { + blocks = append(blocks, contextBlock(line)) + } + return blocks +} + +// terminalBodyBlocks renders the closed root's required sections: final +// outcome, what AlertINT investigated/concluded, duration and peak +// Attention, and recorded operator involvement. It never renders +// ActionContract.NextActor (a terminal contract's own next_actor is not +// the operator-involvement question the spec asks for), so it can never +// collapse to the literal "Actor: none" the spec forbids; FinalOutcome +// already carries Task 4's exact "no recorded operator intervention" +// wording when RecordedOperatorContext is empty. +func terminalBodyBlocks(s model.EpisodeSummary, t model.Transition) []slacklib.Block { + var blocks []slacklib.Block + if line := impactAndOutcomeLine(s); line != "" { + blocks = append(blocks, sectionBlock(line)) + } + if line := investigationLine(s); line != "" { + blocks = append(blocks, sectionBlock(line)) + } + if s.EvidenceConclusion != "" { + blocks = append(blocks, sectionBlock("*Evidence conclusion:* "+s.EvidenceConclusion)) + } + if cl := causalityLine(t.Projection.Assessment); cl != "" { + blocks = append(blocks, sectionBlock(cl)) + } + blocks = append(blocks, sectionBlock(durationPeakLine(s))) + if s.RecoveryObservedAt != nil { + blocks = append(blocks, contextBlock("Recovery observed "+SlackDateToken(*s.RecoveryObservedAt, "{date_short} {time}"))) + } + if line := recurrenceLine(s); line != "" { + blocks = append(blocks, contextBlock(line)) + } + if len(s.RecordedOperatorContext) > 0 { + blocks = append(blocks, sectionBlock("*Recorded operator context:*\n"+boundedList(s.RecordedOperatorContext))) + } + if s.RemainingUncertainty != "" { + blocks = append(blocks, sectionBlock("*Remaining uncertainty:* "+s.RemainingUncertainty)) + } + return blocks +} + +// recurrenceLine reports how many times this Situation's condition has +// recurred, from durable local Store facts alone. Empty when it has never +// recurred. +func recurrenceLine(s model.EpisodeSummary) string { + if s.RecurrenceCount <= 0 { + return "" + } + return fmt.Sprintf(":repeat: recurred ×%d", s.RecurrenceCount) +} + +// impactAndOutcomeLine renders the terminal root's required "what happened +// and the final outcome" section: the recorded impact alongside Task 4's +// own FinalOutcome text (already carrying the exact "recovered without +// recorded operator intervention" wording — spec.md "Root and journal +// rendering" — when no operator context was recorded). +func impactAndOutcomeLine(s model.EpisodeSummary) string { + parts := []string{} + if s.ImpactSummary != "" { + parts = append(parts, s.ImpactSummary) + } + if s.FinalOutcome != "" { + parts = append(parts, s.FinalOutcome) + } + if len(parts) == 0 { + return "" + } + return "*Outcome:* " + strings.Join(parts, " — ") +} + +func whatIsHappeningLine(s model.EpisodeSummary) string { + parts := []string{} + if label := reasonLabel(s.LatestMaterialReason); label != "" { + parts = append(parts, label) + } + if s.ImpactSummary != "" { + parts = append(parts, s.ImpactSummary) + } + if len(parts) == 0 { + return "" + } + return "*What's happening:* " + strings.Join(parts, " — ") +} + +func attentionLine(s model.EpisodeSummary, t model.Transition) string { + parts := []string{"*Attention:* " + humanizeAttention(s.CurrentAttention)} + if s.EvidenceConclusion != "" { + parts = append(parts, s.EvidenceConclusion) + } + if cl := causalityLine(t.Projection.Assessment); cl != "" { + parts = append(parts, cl) + } + return strings.Join(parts, " — ") +} + +func investigationLine(s model.EpisodeSummary) string { + if len(s.InvestigationWork) == 0 { + return "" + } + return "*AlertINT checked:*\n" + boundedList(s.InvestigationWork) +} + +func durationPeakLine(s model.EpisodeSummary) string { + duration := "unknown" + if s.DurationSeconds != nil { + duration = (time.Duration(*s.DurationSeconds) * time.Second).String() + } + return fmt.Sprintf("*Duration:* %s · *Peak attention:* %s", duration, humanizeAttention(s.PeakAttention)) +} + +// causalityLine reports what the evidence supports without ever +// overclaiming: unknown causality is rendered as "reporting observed +// symptoms and checks only," never as a root-cause conclusion (Step 2: +// "Unknown causality must stay observed symptoms/checks and never render +// as root cause"). +func causalityLine(a *model.AssessmentConclusion) string { + if a == nil { + return "" + } + switch a.Causality { //nolint:exhaustive // CausalityUnknown (and any unrecognized value) is the intentional default below. + case model.CausalitySupported: + return "Evidence supports a specific cause." + case model.CausalityCorrelated: + return "Evidence correlates with a candidate cause, not yet confirmed." + case model.CausalityOperatorConfirmed: + return "Operator confirmed the cause." + case model.CausalityContradicted: + return "Evidence contradicts the suspected cause." + default: // CausalityUnknown and any unrecognized value. + return "Cause not established — reporting observed symptoms and checks only." + } +} + +func handleBlock(s model.EpisodeSummary) slacklib.Block { + handle := s.PublicHandle + if handle == "" { + handle = s.SituationID + } + return contextBlock(fmt.Sprintf(":robot_face: Situation `%s` · `get situation %s using alertint`", handle, handle)) +} + +func rootFallback(s model.EpisodeSummary, o situation.Orientation, drill bool) string { + return fmt.Sprintf("%s%s — %s", drillPlainPrefix(drill), s.Title, string(o)) +} + +func drillPlainPrefix(drill bool) string { + if drill { + return "🧪 DRILL — " + } + return "" +} + +// ---------------------------------------------------------------------- +// Journal +// ---------------------------------------------------------------------- + +// RenderSituationJournal renders one immutable journal entry from its +// referenced Transition alone. Delayed/NoLongerCurrent are read straight +// off t.Journal: Task 4's BuildTransitions never sets either (a Transition +// cannot know at creation time whether a later delivery attempt will find +// its handoff stale), so a caller that has determined at delivery time +// that a broadcast handoff is no longer current renders from a local copy +// of t with those two fields set — the stored ledger row itself is never +// mutated. +func RenderSituationJournal(t model.Transition) (RenderedMessage, error) { + if err := t.Validate(); err != nil { + return RenderedMessage{}, fmt.Errorf("slack: render situation journal: %w", err) + } + if t.JournalKind == model.JournalNone { + return RenderedMessage{}, errors.New("slack: render situation journal: transition carries no journal entry") + } + + prefix := drillPrefix(t.Drill) + headline := prefix + "*" + t.Journal.Headline + "*" + blocks := []slacklib.Block{sectionBlock(headline)} + if t.Journal.Detail != "" { + blocks = append(blocks, sectionBlock(t.Journal.Detail)) + } + + var markers []string + if t.Journal.NoLongerCurrent { + markers = append(markers, "no longer current") + } + if t.Journal.Delayed { + markers = append(markers, "delayed") + } + if len(markers) > 0 { + blocks = append(blocks, contextBlock(":clock3: "+strings.Join(markers, " · "))) + } + blocks = append(blocks, contextBlock(SlackDateToken(t.Journal.OccurredAt, "{date_short} {time}"))) + + return RenderedMessage{ + Text: prefix + t.Journal.Headline, + Blocks: blocks, + }, nil +} + +// ---------------------------------------------------------------------- +// Installation Delivery-gap recovery notice +// ---------------------------------------------------------------------- + +// GapNoticeInput is the exact durable gap generation +// RenderDeliveryGapNotice renders one installation_gap_recovery notice +// from — opened_at, recovered_at, and the affected/delayed counts +// recorded on the gap generation (spec.md "Recovery replay": "reports the +// gap interval, affected Situation count, delayed effect count, and +// backlog delivery status"). +type GapNoticeInput struct { + GapID string + OpenedAt time.Time + RecoveredAt time.Time + AffectedSituationCount int + DelayedEffectCount int +} + +// RenderDeliveryGapNotice renders the one bounded ADR-0042 System message +// that precedes backlog replay. It is plain text, matching +// PostSystemMessage's own convention for an installation-level notice — +// this is not a Situation card and carries no Block Kit body. +func RenderDeliveryGapNotice(in GapNoticeInput) (RenderedMessage, error) { + if strings.TrimSpace(in.GapID) == "" { + return RenderedMessage{}, errors.New("slack: render delivery gap notice: gap id is required") + } + if in.OpenedAt.IsZero() { + return RenderedMessage{}, errors.New("slack: render delivery gap notice: opened_at is required") + } + if in.RecoveredAt.IsZero() { + return RenderedMessage{}, errors.New("slack: render delivery gap notice: recovered_at is required") + } + if in.RecoveredAt.Before(in.OpenedAt) { + return RenderedMessage{}, errors.New("slack: render delivery gap notice: recovered_at precedes opened_at") + } + if in.AffectedSituationCount < 0 || in.DelayedEffectCount < 0 { + return RenderedMessage{}, errors.New("slack: render delivery gap notice: counts must be >= 0") + } + duration := in.RecoveredAt.Sub(in.OpenedAt).Round(time.Second) + text := fmt.Sprintf( + ":warning: AlertINT's Slack delivery was interrupted from %s to %s (%s). %d Situation(s) affected, %d delayed effect(s). Replaying the backlog now.", + SlackDateToken(in.OpenedAt, "{date_short} {time}"), + SlackDateToken(in.RecoveredAt, "{date_short} {time}"), + duration, in.AffectedSituationCount, in.DelayedEffectCount) + return RenderedMessage{Text: text}, nil +} + +// ---------------------------------------------------------------------- +// Small rendering helpers +// ---------------------------------------------------------------------- + +func drillPrefix(drill bool) string { + if drill { + return ":test_tube: *DRILL* — " + } + return "" +} + +func humanizeAttention(a model.Attention) string { + switch a { + case model.AttentionObserve: + return "observe" + case model.AttentionInvestigate: + return "investigate" + case model.AttentionUrgent: + return "urgent" + default: + return string(a) + } +} + +// reasonLabel humanizes a closed TransitionReason code for the "what's +// happening" line. It never invents facts beyond the code itself. +func reasonLabel(reason string) string { + switch model.TransitionReason(reason) { + case model.ReasonFirstAuthoritativeState: + return "Situation published" + case model.ReasonMaterialAssessmentChanged: + return "Assessment changed" + case model.ReasonAttentionChanged: + return "Attention changed" + case model.ReasonOperatorContractChanged: + return "Operator contract changed" + case model.ReasonInvestigationStarted: + return "Investigation started" + case model.ReasonInvestigationConcluded: + return "Investigation concluded" + case model.ReasonRecoveryObserved: + return "Recovery observed" + case model.ReasonRecoveryFailed: + return "Recovery did not hold" + case model.ReasonRecovered: + return "Recovered" + case model.ReasonClosedUnknown: + return "Closed with uncertainty" + case model.ReasonRecurrenceMilestone: + return "Recurrence milestone" + case model.ReasonTriageStateChanged: + return "Acute Triage state changed" + case model.ReasonOperatorArtifactRecorded: + return "Operator context recorded" + default: + return "" + } +} + +// contractLine renders the compact Operator-contract action, e.g. "AlertINT +// is running Acute Triage" or "Watching for sustained recovery." Monitoring +// always reads as watching for recovery regardless of the underlying +// contract's own fields — the phase itself already says why. +func contractLine(o situation.Orientation, c model.ActionContract) string { + if o == situation.OrientationMonitoring { + return "Watching for sustained recovery" + } + switch { + case c.OperatorActionRequired != nil: + return "Operator action required: " + humanizeOperatorAction(*c.OperatorActionRequired) + case c.AlertINTAction != nil: + return humanizeAlertINTAction(*c.AlertINTAction, c.AlertINTStatus) + default: + return "No action currently required" + } +} + +func humanizeOperatorAction(a model.OperatorAction) string { + switch a { + case model.OperatorActionInvestigateSituation: + return "investigate this Situation" + default: + return string(a) + } +} + +func humanizeAlertINTAction(a model.AlertINTAction, status *model.AlertINTStatus) string { + verb := "is running" + if status != nil { + switch *status { + case model.AlertINTStatusPlanned: + verb = "will run" + case model.AlertINTStatusRunning: + verb = "is running" + case model.AlertINTStatusWaiting: + verb = "is waiting on" + case model.AlertINTStatusBlocked: + verb = "is blocked on" + case model.AlertINTStatusExhausted: + verb = "has exhausted its attempts at" + case model.AlertINTStatusComplete: + verb = "has completed" + } + } + return fmt.Sprintf("AlertINT %s %s", verb, humanizeAlertINTActionLabel(a)) +} + +func humanizeAlertINTActionLabel(a model.AlertINTAction) string { + switch a { + case model.AlertINTActionRunAcuteTriage: + return "Acute Triage" + case model.AlertINTActionRetrySituationAssessment: + return "its assessment" + case model.AlertINTActionMonitorSituation: + return "monitoring" + case model.AlertINTActionVerifyRecovery: + return "recovery verification" + default: + return string(a) + } +} + +// sectionBlock and contextBlock both apply the same bounded truncation with +// a visible marker (Step 3) rather than exceeding Slack's own per-block +// text limit. +func sectionBlock(text string) slacklib.Block { + return slacklib.NewSectionBlock( + slacklib.NewTextBlockObject(slacklib.MarkdownType, truncateText(text), false, false), + nil, nil) +} + +func contextBlock(text string) slacklib.Block { + return slacklib.NewContextBlock("", + slacklib.NewTextBlockObject(slacklib.MarkdownType, truncateText(text), false, false)) +} + +func truncateText(s string) string { + if len(s) <= maxSectionChars { + return s + } + cut := maxSectionChars - len(truncationMarker) - 1 + if cut < 0 { + cut = 0 + } + return boundedRunes(s, cut) + " " + truncationMarker +} + +// boundedRunes truncates s to at most limit bytes without splitting a rune +// (mirrors internal/situation/history.go's own boundedText, unexported +// there). +func boundedRunes(s string, limit int) string { + if len(s) <= limit { + return s + } + cut := s[:limit] + for len(cut) > 0 && !utf8.ValidString(cut) { + cut = cut[:len(cut)-1] + } + return cut +} + +// boundedList renders the most recent maxRenderedListEntries of entries as +// a bullet list, oldest of the shown entries first, with a leading visible +// marker naming how many earlier entries were not shown — never a silent +// drop. +func boundedList(entries []string) string { + shown := entries + omitted := 0 + if len(entries) > maxRenderedListEntries { + omitted = len(entries) - maxRenderedListEntries + shown = entries[omitted:] + } + lines := make([]string, 0, len(shown)+1) + if omitted > 0 { + lines = append(lines, fmt.Sprintf("… %d earlier %s not shown", omitted, pluralize(omitted, "entry", "entries"))) + } + for _, e := range shown { + lines = append(lines, "• "+e) + } + return strings.Join(lines, "\n") +} + +func pluralize(n int, singular, plural string) string { + if n == 1 { + return singular + } + return plural +} diff --git a/internal/notify/slack/situation_test.go b/internal/notify/slack/situation_test.go new file mode 100644 index 0000000..334af31 --- /dev/null +++ b/internal/notify/slack/situation_test.go @@ -0,0 +1,816 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package slack + +import ( + "fmt" + "strings" + "testing" + "time" + + slacklib "github.com/slack-go/slack" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Fixtures. Prefixed `rs` (render situation) so they never collide with +// any package-level helper this file's sibling test files add. +// ---------------------------------------------------------------------- + +const rsSituationID = "1f0f5a0c-0000-4000-8000-0000000000a1" + +func rsMustTime(t *testing.T, s string) time.Time { + t.Helper() + tm, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatalf("parse time %q: %v", s, err) + } + return tm.UTC() +} + +func rsTimePtr(t time.Time) *time.Time { return &t } + +func rsRunningTriageContract(next time.Time) model.ActionContract { + action := model.AlertINTActionRunAcuteTriage + status := model.AlertINTStatusRunning + return model.ActionContract{ + NextActor: model.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: rsTimePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnTriageOutcome}, + } +} + +func rsMonitoringContract(next time.Time) model.ActionContract { + action := model.AlertINTActionVerifyRecovery + status := model.AlertINTStatusWaiting + wait := model.WaitReasonRecoveryGrace + return model.ActionContract{ + NextActor: model.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: rsTimePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnRecoveryGraceExpired}, + WaitReason: &wait, + } +} + +func rsTerminalContract() model.ActionContract { + return model.ActionContract{NextActor: model.NextActorNone} +} + +// rsObserveContract is a valid nonterminal contract with no current +// AlertINT action or Operator ask — the pre-investigation "just observing" +// state. +func rsObserveContract(next time.Time) model.ActionContract { + return model.ActionContract{ + NextActor: model.NextActorNone, + NextUpdateAt: rsTimePtr(next), + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnLifecycleObservationDeadline}, + } +} + +func rsAssessment(causality model.Causality, impact model.Impact) *model.AssessmentConclusion { + return &model.AssessmentConclusion{ + Persistence: model.PersistenceSustained, + Impact: impact, + Novelty: model.NoveltyFamiliar, + Causality: causality, + EvidenceQuality: model.EvidenceQualityComplete, + SufficientReasonSummary: "checkout latency exceeded threshold for 10 minutes", + } +} + +func rsProjection(startedAt time.Time, assessment *model.AssessmentConclusion) model.ProjectionFacts { + return model.ProjectionFacts{ + EffectiveStartedAt: startedAt, + EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload, + Assessment: assessment, + } +} + +// rsTransition builds a valid Transition fixture. Callers mutate the +// returned value's Projection/Journal fields directly for scenarios that +// need extra facts (recovery instants, terminal reason, drill). +func rsTransition(seq int, lifecycle model.Lifecycle, attention model.Attention, contract model.ActionContract, + reason model.TransitionReason, journalKind model.JournalKind, journal model.JournalData, + projection model.ProjectionFacts, createdAt time.Time) model.Transition { + return model.Transition{ + ID: fmt.Sprintf("transition-%03d", seq), + SituationID: rsSituationID, + Sequence: seq, + InputVersion: seq, + MaterialFactHash: "sha256:abc123", + Lifecycle: lifecycle, + Attention: attention, + ActionContract: contract, + Reason: reason, + JournalKind: journalKind, + Journal: journal, + Projection: projection, + EvidenceRefs: []string{"evidence-1"}, + Actor: model.ActorDeterministicController, + CreatedAt: createdAt, + } +} + +// rsSummary builds a valid EpisodeSummary fixture coherent with a +// Transition of the given sequence. +func rsSummary(seq int, title string, attention model.Attention, contract model.ActionContract, + startedAt, updatedAt time.Time) model.EpisodeSummary { + return model.EpisodeSummary{ + SituationID: rsSituationID, + Version: seq, + SourceTransitionSequence: seq, + Title: title, + CurrentAttention: attention, + PeakAttention: attention, + ActionContract: contract, + EffectiveStartedAt: startedAt, + UpdatedAt: updatedAt, + InvestigationWork: []string{}, + RecordedOperatorContext: []string{}, + } +} + +func rsFallbackBlocksText(msg RenderedMessage) string { + var b strings.Builder + for _, blk := range msg.Blocks { + switch v := blk.(type) { + case *slacklib.SectionBlock: + if v.Text != nil { + b.WriteString(v.Text.Text) + b.WriteString("\n") + } + case *slacklib.ContextBlock: + for _, el := range v.ContextElements.Elements { + if txt, ok := el.(*slacklib.TextBlockObject); ok { + b.WriteString(txt.Text) + b.WriteString("\n") + } + } + } + } + return b.String() +} + +func rsCountBold(s, phase string) int { + return strings.Count(s, "*"+phase+"*") +} + +// ---------------------------------------------------------------------- +// Step 1: first/updated root, orientation, R4 deadline rendering. +// ---------------------------------------------------------------------- + +func TestRenderSituationRootFirstAndUpdated(t *testing.T) { + started := rsMustTime(t, "2026-09-05T09:00:00Z") + now := rsMustTime(t, "2026-09-05T10:00:00Z") + deadline := now.Add(90 * time.Second) // ceil-rounds to 2 minutes + + cases := []struct { + name string + lifecycle model.Lifecycle + attention model.Attention + reason model.TransitionReason + contract model.ActionContract + investigation []string + wantBoldPhase string + wantOrient string + wantContract string + }{ + { + name: "first publication, before investigation", + lifecycle: model.LifecycleActive, + attention: model.AttentionObserve, + reason: model.ReasonFirstAuthoritativeState, + contract: rsObserveContract(deadline), + wantBoldPhase: "Observed", + wantOrient: "observed", + wantContract: "No action currently required", + }, + { + name: "updated after investigation starts", + lifecycle: model.LifecycleActive, + attention: model.AttentionInvestigate, + reason: model.ReasonInvestigationStarted, + contract: rsRunningTriageContract(deadline), + investigation: []string{"ran acute triage"}, + wantBoldPhase: "Investigating", + wantOrient: "investigating", + wantContract: "AlertINT is running Acute Triage", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + contract := c.contract + tr := rsTransition(1, c.lifecycle, c.attention, contract, c.reason, + model.JournalPublication, model.JournalData{Headline: "Situation published", OccurredAt: started}, + rsProjection(started, rsAssessment(model.CausalitySupported, model.ImpactConfirmed)), now) + summary := rsSummary(1, "Situation checkout-001", c.attention, contract, started, now) + summary.InvestigationStarted = len(c.investigation) > 0 + summary.InvestigationWork = c.investigation + summary.EvidenceConclusion = "checkout latency exceeded threshold" + summary.ImpactSummary = "Confirmed impact" + summary.LatestMaterialReason = string(c.reason) + + got, err := RenderSituationRoot(SituationRootInput{ + Summary: summary, + SourceTransition: tr, + ContractDeadlineAt: &deadline, + Now: now, + }) + if err != nil { + t.Fatalf("RenderSituationRoot() error = %v", err) + } + text := rsFallbackBlocksText(got) + + if rsCountBold(text, c.wantBoldPhase) != 1 { + t.Fatalf("orientation chain = %q, want exactly one bold phase %q", text, c.wantBoldPhase) + } + // Exactly one bolded phase overall (no other phase word wrapped + // in "*...*"). + totalBoldMarkers := strings.Count(text, "*Observed*") + strings.Count(text, "*Investigating*") + + strings.Count(text, "*Monitoring*") + strings.Count(text, "*Recovered*") + strings.Count(text, "*Closed uncertain*") + if totalBoldMarkers != 1 { + t.Fatalf("expected exactly one bolded orientation phase, got %d in %q", totalBoldMarkers, text) + } + if !strings.Contains(got.Text, c.wantOrient) { + t.Fatalf("fallback text = %q, want to mention orientation %q", got.Text, c.wantOrient) + } + if !strings.Contains(text, c.wantContract) { + t.Fatalf("body = %q, want the compact contract line %q", text, c.wantContract) + } + if !strings.Contains(text, "update by") || !strings.Contains(text, "(2 min)") { + t.Fatalf("body = %q, want a ceil-rounded 2-minute countdown", text) + } + if !strings.Contains(text, " token, for a surface that cannot render Slack's special +// markup (spec.md "Root and journal rendering": "Every instant uses Slack +// viewer-local date markup with a UTC fallback"). +const slackDateFallbackLayout = "2006-01-02 15:04 MST" + +// SlackDateToken renders t as a Slack mrkdwn date token: the viewer's own +// client renders it in the viewer's local time zone and format; a client +// that cannot (a plain-text notification, a screen reader, a webhook log) +// falls back to the fixed UTC string. format is a Slack date-format +// string, e.g. "{date_short} {time}" or "{time}"; see +// https://api.slack.com/reference/surfaces/formatting#date-formatting. +func SlackDateToken(t time.Time, format string) string { + u := t.UTC() + return fmt.Sprintf("", u.Unix(), format, u.Format(slackDateFallbackLayout)) +} + +// CeilMinutes rounds d up to the nearest whole minute, never negative — the +// promised-update countdown's ceil-rounded minutes remaining (R4). +func CeilMinutes(d time.Duration) int64 { + if d <= 0 { + return 0 + } + return int64((d + time.Minute - 1) / time.Minute) +} + +// RenderDeadline renders one Operator-contract promised-update instant +// relative to now: while still in the future, a countdown ("update by +// (N min)"); once reached or passed at render time, "update +// overdue since " rather than a current promise (R4: "A deadline +// already past at render time renders as overdue, never as a current +// promise"). +func RenderDeadline(deadline, now time.Time) string { + if !deadline.After(now) { + return fmt.Sprintf("update overdue since %s", SlackDateToken(deadline, "{date_short_pretty} {time}")) + } + minutes := CeilMinutes(deadline.Sub(now)) + return fmt.Sprintf("update by %s (%d min)", SlackDateToken(deadline, "{time}"), minutes) +} diff --git a/internal/notify/slack/time_test.go b/internal/notify/slack/time_test.go new file mode 100644 index 0000000..99c5138 --- /dev/null +++ b/internal/notify/slack/time_test.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package slack + +import ( + "strconv" + "strings" + "testing" + "time" +) + +func TestSlackAPISlackDateTokenFormat(t *testing.T) { + // A fixed instant in a non-UTC location: the token's Unix stamp and UTC + // fallback must both be computed from the UTC instant, never the + // location the caller happened to pass in. + loc := time.FixedZone("TEST", 3*60*60) // UTC+3 + instant := time.Date(2026, 9, 5, 13, 4, 5, 0, loc) + utc := instant.UTC() + + got := SlackDateToken(instant, "{date_short} {time}") + + wantPrefix := "") { + t.Fatalf("SlackDateToken() = %q, want UTC fallback suffix for %s", got, utc) + } + if !strings.Contains(got, "UTC") { + t.Fatalf("SlackDateToken() = %q, want a UTC fallback marker", got) + } +} + +func TestSlackAPICeilMinutes(t *testing.T) { + cases := []struct { + name string + d time.Duration + want int64 + }{ + {"zero", 0, 0}, + {"negative", -30 * time.Second, 0}, + {"exact minute", 2 * time.Minute, 2}, + {"one second over", 2*time.Minute + time.Second, 3}, + {"sub-minute remainder", 30 * time.Second, 1}, + {"just under a minute", 59 * time.Second, 1}, + {"large", 100*time.Minute + time.Nanosecond, 101}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := CeilMinutes(c.d); got != c.want { + t.Fatalf("CeilMinutes(%s) = %d, want %d", c.d, got, c.want) + } + }) + } +} + +func TestSlackAPIRenderDeadlineFuture(t *testing.T) { + now := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + deadline := now.Add(90 * time.Second) // ceil-rounds to 2 minutes + + got := RenderDeadline(deadline, now) + + if !strings.HasPrefix(got, "update by ") { + t.Fatalf("RenderDeadline() = %q, want a current promise, not overdue", got) + } + if !strings.Contains(got, "(2 min)") { + t.Fatalf("RenderDeadline() = %q, want ceil-rounded 2 min remaining", got) + } + if !strings.Contains(got, " Date: Sun, 6 Sep 2026 05:05:49 +0300 Subject: [PATCH 10/31] feat(situation): deliver durable Slack history Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- cmd/alertint/situation_notifications.go | 221 +++-- cmd/alertint/situation_notifications_test.go | 13 +- internal/situation/notification_worker.go | 907 ++++++++++++++++++ .../situation/notification_worker_test.go | 749 +++++++++++++++ internal/store/notification_gaps.go | 449 +++++++++ internal/store/notification_gaps_test.go | 344 +++++++ internal/store/situation_notifications.go | 578 +++++++++++ .../store/situation_notifications_test.go | 638 ++++++++++++ 8 files changed, 3815 insertions(+), 84 deletions(-) create mode 100644 internal/situation/notification_worker.go create mode 100644 internal/situation/notification_worker_test.go create mode 100644 internal/store/notification_gaps.go create mode 100644 internal/store/notification_gaps_test.go create mode 100644 internal/store/situation_notifications.go create mode 100644 internal/store/situation_notifications_test.go diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go index a33d436..9bf35b1 100644 --- a/cmd/alertint/situation_notifications.go +++ b/cmd/alertint/situation_notifications.go @@ -9,6 +9,7 @@ import ( "time" "github.com/alertint/alertint-agent/internal/notify/slack" + "github.com/alertint/alertint-agent/internal/situation" "github.com/alertint/alertint-agent/internal/situation/model" "github.com/alertint/alertint-agent/internal/store" ) @@ -25,50 +26,33 @@ import ( // tracking, and acknowledging the result back into the Store are Task 7's // notification worker, not this file. // -// NotificationDelivery below is a TEMPORARY stand-in. The plan's -// Cross-Task Contracts define `NotificationDelivery{Channel, MessageTS, -// DeliveredAs}` and `NotificationDeliverer{Probe, Deliver}` in -// internal/situation/notification_worker.go — which is Task 7's file and -// does not exist yet. This type carries EXACTLY those field names and -// shapes so Task 7's dispatch can replace this file's `NotificationDelivery` -// references with `situation.NotificationDelivery` (and delete this local -// definition) as a mechanical rename, with no other change to -// SituationDeliverer's logic expected. +// Task 7 alignment: the delivery result and deliverer contract now live +// where the plan's Cross-Task Contracts put them — +// situation.NotificationDelivery and situation.NotificationDeliverer — and +// the gap-rendering snapshot is store.GapSnapshot; this file's own +// placeholders for all three are retired. SituationDeliverer's rendering +// and Slack-call logic is unchanged by that alignment. What it gained is +// classifyDeliveryError below, which translates one failed call into the +// closed situation.DeliveryFailure classification the worker resolves its +// retry / configuration-block / fail outcome from. That translation lives +// here on purpose: this package already owns the Slack wire, so +// internal/situation stays free of any Slack dependency. // ---------------------------------------------------------------------- -// NotificationDelivery is the durable Slack coordinate and delivery shape -// one Deliver call returns. DeliveredAs is one of: root | thread | -// broadcast | delayed_thread | system. -type NotificationDelivery struct { - Channel string - MessageTS string - DeliveredAs string -} - -// GapSnapshot is the durable installation-level Delivery-gap generation -// RenderDeliveryGapNotice renders one recovery notice from: opened_at, -// recovered_at, and the affected/delayed counts recorded on -// slack_delivery_gaps (migration 0018). No Task 5 reader exposes this -// table today (Task 5's situation_views.go covers Situation/Transition/ -// intent reads only); Task 7's internal/store/notification_gaps.go is -// expected to add a matching GetDeliveryGap(ctx, id) (GapSnapshot, error) -// reader on *store.Store so it satisfies DelivererStore unchanged. -type GapSnapshot struct { - ID string - OpenedAt time.Time - RecoveredAt time.Time - AffectedSituationCount int - DelayedEffectCount int -} +// Compile-time proof of the assembly this task closes: the Slack adapter is +// a situation.NotificationDeliverer, and *store.Store satisfies both this +// file's reader contract and the worker's whole durable contract. +var ( + _ situation.NotificationDeliverer = (*SituationDeliverer)(nil) + _ DelivererStore = (*store.Store)(nil) + _ situation.NotificationStore = (*store.Store)(nil) +) // DelivererStore is exactly what SituationDeliverer reads. The first three -// methods are Task 5's existing bounded readers -// (internal/store/situation_views.go) — *store.Store already satisfies -// them. The last two are new readers this task's design found missing (see -// their own doc comments): Task 7 (or a follow-up to Task 5) is expected to -// add them to *store.Store with exactly these signatures so *store.Store -// satisfies DelivererStore unchanged; this task's own tests exercise -// SituationDeliverer against a hand-rolled fake instead. +// methods are Task 5's bounded readers (internal/store/situation_views.go); +// the last two are Task 7's (internal/store/situation_notifications.go and +// internal/store/notification_gaps.go). *store.Store satisfies all five — +// asserted above. type DelivererStore interface { GetSituationEpisodeView(ctx context.Context, situationID string) (store.SituationEpisodeView, error) GetSituationTransition(ctx context.Context, transitionID string) (model.Transition, error) @@ -80,7 +64,7 @@ type DelivererStore interface { GetSituationRootCoordinates(ctx context.Context, situationID string) (channel, messageTS string, ok bool, err error) // GetDeliveryGap reads one durable gap generation's rendering facts. - GetDeliveryGap(ctx context.Context, gapGeneration string) (GapSnapshot, error) + GetDeliveryGap(ctx context.Context, gapGeneration string) (store.GapSnapshot, error) } // slackDeliveryAPI is exactly what SituationDeliverer calls on the narrow @@ -130,9 +114,20 @@ func (d *SituationDeliverer) Probe(ctx context.Context) error { // exact durable records it references through DelivererStore. It never // decides whether a root is durably delivered before a reply is claimable // (Task 7's ordering) and never writes Store state itself. -func (d *SituationDeliverer) Deliver(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { +func (d *SituationDeliverer) Deliver(ctx context.Context, intent model.NotificationIntent) (situation.NotificationDelivery, error) { + delivery, err := d.deliver(ctx, intent) + if err != nil { + return situation.NotificationDelivery{}, classifyDeliveryError(err) + } + return delivery, nil +} + +// deliver is Deliver's undecorated body: it renders and sends, and returns +// raw errors that Deliver classifies on the way out. +func (d *SituationDeliverer) deliver(ctx context.Context, intent model.NotificationIntent) (situation.NotificationDelivery, error) { if err := intent.Validate(); err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: %w", err) + return situation.NotificationDelivery{}, invalidDelivery("invalid_intent", + fmt.Errorf("cmd/alertint: situation deliverer: %w", err)) } switch intent.EffectClass { case model.EffectRootSync: @@ -144,7 +139,8 @@ func (d *SituationDeliverer) Deliver(ctx context.Context, intent model.Notificat case model.EffectInstallationGapRecovery: return d.deliverGapRecovery(ctx, intent) default: - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: unknown effect class %q", intent.EffectClass) + return situation.NotificationDelivery{}, invalidDelivery("unknown_effect_class", + fmt.Errorf("cmd/alertint: situation deliverer: unknown effect class %q", intent.EffectClass)) } } @@ -152,22 +148,23 @@ func (d *SituationDeliverer) Deliver(ctx context.Context, intent model.Notificat // (including the R4 deadline refresh, which is a plain chat.update): the // selected Episode-summary version renders only from GetSituationEpisodeView's // own coherent (summary, source Transition) pair. -func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { +func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.NotificationIntent) (situation.NotificationDelivery, error) { if intent.SituationID == nil || intent.SummaryVersion == nil { - return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: root_sync intent missing situation_id/summary_version") + return situation.NotificationDelivery{}, invalidDelivery("incomplete_intent", + errors.New("cmd/alertint: situation deliverer: root_sync intent missing situation_id/summary_version")) } view, err := d.store.GetSituationEpisodeView(ctx, *intent.SituationID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) } if view.Summary.Version != *intent.SummaryVersion { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: intent names summary version %d, current is %d", + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: intent names summary version %d, current is %d", *intent.SummaryVersion, view.Summary.Version) } recoveryEverObserved, err := d.recoveryEverObserved(ctx, view) if err != nil { - return NotificationDelivery{}, err + return situation.NotificationDelivery{}, err } rendered, err := slack.RenderSituationRoot(slack.SituationRootInput{ @@ -178,12 +175,13 @@ func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.N RecoveryEverObserved: recoveryEverObserved, }) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render root: %w", err) + return situation.NotificationDelivery{}, invalidDelivery("render_failed", + fmt.Errorf("cmd/alertint: situation deliverer: render root: %w", err)) } channel, ts, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) } if !ok { res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ @@ -193,9 +191,9 @@ func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.N ClientMsgID: intent.ClientMessageID, }) if err != nil { - return NotificationDelivery{}, err + return situation.NotificationDelivery{}, err } - return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "root"}, nil + return situation.NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "root"}, nil } res, err := d.api.UpdateMessage(ctx, slack.UpdateMessageRequest{ Channel: channel, @@ -205,32 +203,34 @@ func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.N ClientMsgID: intent.ClientMessageID, }) if err != nil { - return NotificationDelivery{}, err + return situation.NotificationDelivery{}, err } - return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "root"}, nil + return situation.NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "root"}, nil } // deliverThreadAppend appends one immutable journal entry to the // Situation's existing root thread, rendering only from its own referenced // Transition. -func (d *SituationDeliverer) deliverThreadAppend(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { +func (d *SituationDeliverer) deliverThreadAppend(ctx context.Context, intent model.NotificationIntent) (situation.NotificationDelivery, error) { if intent.SituationID == nil || intent.TransitionID == nil { - return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: thread_append intent missing situation_id/transition_id") + return situation.NotificationDelivery{}, invalidDelivery("incomplete_intent", + errors.New("cmd/alertint: situation deliverer: thread_append intent missing situation_id/transition_id")) } tr, err := d.store.GetSituationTransition(ctx, *intent.TransitionID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) } channel, rootTS, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) } if !ok { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) } rendered, err := slack.RenderSituationJournal(tr) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render journal: %w", err) + return situation.NotificationDelivery{}, invalidDelivery("render_failed", + fmt.Errorf("cmd/alertint: situation deliverer: render journal: %w", err)) } res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ Channel: channel, @@ -240,9 +240,9 @@ func (d *SituationDeliverer) deliverThreadAppend(ctx context.Context, intent mod ClientMsgID: intent.ClientMessageID, }) if err != nil { - return NotificationDelivery{}, err + return situation.NotificationDelivery{}, err } - return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "thread"}, nil + return situation.NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "thread"}, nil } // deliverBroadcastHandoff optionally broadcasts a current handoff. @@ -252,24 +252,25 @@ func (d *SituationDeliverer) deliverThreadAppend(ctx context.Context, intent mod // newer Transition has since superseded it, the same Transition is // delivered instead as a plain, delayed, no-longer-current thread reply — // never a channel broadcast. -func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { +func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent model.NotificationIntent) (situation.NotificationDelivery, error) { if intent.SituationID == nil || intent.TransitionID == nil { - return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: broadcast_handoff intent missing situation_id/transition_id") + return situation.NotificationDelivery{}, invalidDelivery("incomplete_intent", + errors.New("cmd/alertint: situation deliverer: broadcast_handoff intent missing situation_id/transition_id")) } tr, err := d.store.GetSituationTransition(ctx, *intent.TransitionID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) } channel, rootTS, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) } if !ok { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) } view, err := d.store.GetSituationEpisodeView(ctx, *intent.SituationID) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) } current := view.Summary.SourceTransitionSequence == tr.Sequence @@ -280,7 +281,8 @@ func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent } rendered, err := slack.RenderSituationJournal(renderTr) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render journal: %w", err) + return situation.NotificationDelivery{}, invalidDelivery("render_failed", + fmt.Errorf("cmd/alertint: situation deliverer: render journal: %w", err)) } res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ @@ -292,25 +294,26 @@ func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent ClientMsgID: intent.ClientMessageID, }) if err != nil { - return NotificationDelivery{}, err + return situation.NotificationDelivery{}, err } deliveredAs := "broadcast" if !current { deliveredAs = "delayed_thread" } - return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: deliveredAs}, nil + return situation.NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: deliveredAs}, nil } // deliverGapRecovery posts the one bounded installation recovery notice for // gap generation intent.GapGeneration names, rendering only from that // generation's own durable facts. -func (d *SituationDeliverer) deliverGapRecovery(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { +func (d *SituationDeliverer) deliverGapRecovery(ctx context.Context, intent model.NotificationIntent) (situation.NotificationDelivery, error) { if intent.GapGeneration == nil { - return NotificationDelivery{}, errors.New("cmd/alertint: situation deliverer: installation_gap_recovery intent missing gap_generation") + return situation.NotificationDelivery{}, invalidDelivery("incomplete_intent", + errors.New("cmd/alertint: situation deliverer: installation_gap_recovery intent missing gap_generation")) } gap, err := d.store.GetDeliveryGap(ctx, *intent.GapGeneration) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load delivery gap: %w", err) + return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load delivery gap: %w", err) } rendered, err := slack.RenderDeliveryGapNotice(slack.GapNoticeInput{ GapID: gap.ID, @@ -320,7 +323,8 @@ func (d *SituationDeliverer) deliverGapRecovery(ctx context.Context, intent mode DelayedEffectCount: gap.DelayedEffectCount, }) if err != nil { - return NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: render gap notice: %w", err) + return situation.NotificationDelivery{}, invalidDelivery("render_failed", + fmt.Errorf("cmd/alertint: situation deliverer: render gap notice: %w", err)) } res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ Channel: d.channel, @@ -329,9 +333,9 @@ func (d *SituationDeliverer) deliverGapRecovery(ctx context.Context, intent mode ClientMsgID: intent.ClientMessageID, }) if err != nil { - return NotificationDelivery{}, err + return situation.NotificationDelivery{}, err } - return NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "system"}, nil + return situation.NotificationDelivery{Channel: res.Channel, MessageTS: res.TS, DeliveredAs: "system"}, nil } // recoveryEverObserved answers SituationRootInput.RecoveryEverObserved: it @@ -366,3 +370,64 @@ func (d *SituationDeliverer) recoveryEverObserved(ctx context.Context, view stor return false, fmt.Errorf("cmd/alertint: situation deliverer: transition ledger for %s exceeds %d pages", view.Summary.SituationID, maxLedgerScanPages) } + +// ---------------------------------------------------------------------- +// Failure classification (Task 7 alignment). +// ---------------------------------------------------------------------- + +// deliveryAdapterError carries one classified delivery failure across the +// package boundary as a situation.DeliveryFailure, so the notification +// worker resolves retry / configuration-block / fail without ever +// importing internal/notify/slack. +type deliveryAdapterError struct { + class situation.DeliveryErrorClass + code string + retryAfter time.Duration + err error +} + +func (e *deliveryAdapterError) Error() string { return e.err.Error() } +func (e *deliveryAdapterError) Unwrap() error { return e.err } + +func (e *deliveryAdapterError) DeliveryErrorClass() situation.DeliveryErrorClass { return e.class } +func (e *deliveryAdapterError) DeliveryErrorCode() string { return e.code } +func (e *deliveryAdapterError) DeliveryRetryAfter() time.Duration { return e.retryAfter } + +// invalidDelivery marks one of this adapter's own errors as a +// non-recoverable programming/data error: a durable intent this build +// cannot render or send at all, however many times it retries. +func invalidDelivery(code string, err error) error { + return &deliveryAdapterError{class: situation.DeliveryInvalid, code: code, err: err} +} + +// classifyDeliveryError resolves one failed Deliver call into the closed +// situation.DeliveryFailure classification. +// +// Slack's own typed classification (slack.APIError) passes straight +// through: retryable transport/5xx/rate-limit/uncertain outcomes keep their +// Retry-After, definite token/scope/channel rejections block on +// configuration, and a malformed payload this build sent is invalid. +// Anything this adapter already proved invalid keeps that verdict. EVERY +// other error — a Store read failure, a stale summary version, a reply +// whose root is not published yet — stays retryable: none of them proves a +// permanent condition, and only a proven one may ever close a durable +// delivery obligation. +func classifyDeliveryError(err error) error { + var adapterErr *deliveryAdapterError + if errors.As(err, &adapterErr) { + return adapterErr + } + var apiErr *slack.APIError + if errors.As(err, &apiErr) { + class := situation.DeliveryRetryable + switch apiErr.Class { + case slack.ErrorClassConfiguration: + class = situation.DeliveryConfigurationBlocking + case slack.ErrorClassInvalid: + class = situation.DeliveryInvalid + case slack.ErrorClassRetryable: + } + return &deliveryAdapterError{class: class, code: apiErr.Code, retryAfter: apiErr.RetryAfter, err: err} + } + return &deliveryAdapterError{class: situation.DeliveryRetryable, code: "delivery_failed", err: err} +} diff --git a/cmd/alertint/situation_notifications_test.go b/cmd/alertint/situation_notifications_test.go index ee24ba5..4194516 100644 --- a/cmd/alertint/situation_notifications_test.go +++ b/cmd/alertint/situation_notifications_test.go @@ -13,6 +13,7 @@ import ( slacklib "github.com/slack-go/slack" "github.com/alertint/alertint-agent/internal/notify/slack" + "github.com/alertint/alertint-agent/internal/situation" "github.com/alertint/alertint-agent/internal/situation/model" "github.com/alertint/alertint-agent/internal/store" ) @@ -177,7 +178,7 @@ type fakeDelivererStore struct { rootOK bool rootErr error - gap GapSnapshot + gap store.GapSnapshot gapErr error } @@ -220,9 +221,9 @@ func (f *fakeDelivererStore) GetSituationRootCoordinates(context.Context, string return f.rootChannel, f.rootTS, f.rootOK, nil } -func (f *fakeDelivererStore) GetDeliveryGap(context.Context, string) (GapSnapshot, error) { +func (f *fakeDelivererStore) GetDeliveryGap(context.Context, string) (store.GapSnapshot, error) { if f.gapErr != nil { - return GapSnapshot{}, f.gapErr + return store.GapSnapshot{}, f.gapErr } return f.gap, nil } @@ -291,7 +292,7 @@ func TestSituationDelivererRootSyncPostsFirstRoot(t *testing.T) { if err != nil { t.Fatalf("Deliver() error = %v", err) } - if got != (NotificationDelivery{Channel: "C-root", MessageTS: "100.1", DeliveredAs: "root"}) { + if got != (situation.NotificationDelivery{Channel: "C-root", MessageTS: "100.1", DeliveredAs: "root"}) { t.Fatalf("Deliver() = %+v, want the posted root coordinates", got) } if len(api.posts) != 1 || len(api.updates) != 0 { @@ -523,7 +524,7 @@ func TestSituationDelivererGapRecoveryPostsSystemNotice(t *testing.T) { recovered := sdMustTime(t, "2026-09-05T09:10:00Z") now := recovered - fs := &fakeDelivererStore{gap: GapSnapshot{ + fs := &fakeDelivererStore{gap: store.GapSnapshot{ ID: "gap-1", OpenedAt: opened, RecoveredAt: recovered, AffectedSituationCount: 2, DelayedEffectCount: 5, }} api := &fakeSlackAPI{postResult: slack.MessageResult{Channel: "C-default", TS: "70.7"}} @@ -534,7 +535,7 @@ func TestSituationDelivererGapRecoveryPostsSystemNotice(t *testing.T) { if err != nil { t.Fatalf("Deliver() error = %v", err) } - if got != (NotificationDelivery{Channel: "C-default", MessageTS: "70.7", DeliveredAs: "system"}) { + if got != (situation.NotificationDelivery{Channel: "C-default", MessageTS: "70.7", DeliveredAs: "system"}) { t.Fatalf("Deliver() = %+v, want the posted system notice coordinates", got) } if len(api.posts) != 1 || api.posts[0].Channel != "C-default" { diff --git a/internal/situation/notification_worker.go b/internal/situation/notification_worker.go new file mode 100644 index 0000000..7cad036 --- /dev/null +++ b/internal/situation/notification_worker.go @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "context" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 7: the Situation notification delivery worker — the single +// reachable Slack writer for Situation history. +// +// It owns exactly four things: claiming durable notification intents under +// a fenced lease, calling the deliverer once per claim, acknowledging the +// real outcome durably, and driving the installation-level Delivery-gap +// state machine (ordinary delay -> open generation -> replaying -> +// complete). It decides no publication policy (that is Task 4's +// BuildTransitions/PlanNotificationIntents inside the authoritative +// commit), renders nothing (Task 6), and never supersedes a root +// projection itself: root supersession happens inside Task 5's fenced +// controller commit, and this worker only ever DETECTS that it lost a +// claim to one (R4). +// +// Valid effects retry indefinitely. There is no attempt ceiling anywhere +// in this file: only an invalid durable intent (a programming/data error +// the deliverer proves) becomes `failed`, and even that is explicitly +// operator-redriveable. +// ---------------------------------------------------------------------- + +var ( + // ErrNotificationClaimLost means a fenced acknowledgement named a + // claim that is no longer the intent's current one: the lease expired + // and was swept or reclaimed, the intent was released, or another + // holder's token superseded this one. The write changed zero rows. + ErrNotificationClaimLost = errors.New("situation: notification claim lost") + + // ErrNotificationIntentSuperseded means the claimed root projection was + // superseded by a newer one inside a concurrent authoritative commit + // while this worker was mid-flight (R4). It is the expected outcome of + // that race, not a delivery failure: the newer projection renders what + // this one would have, and a superseded intent can never become + // delivered. + ErrNotificationIntentSuperseded = errors.New("situation: notification intent superseded") +) + +// NotificationClaim is one durable notification intent leased to this +// worker, together with the fencing pair every acknowledgement must carry. +type NotificationClaim struct { + Intent model.NotificationIntent + ClaimOwner string + ClaimToken int64 +} + +// NotificationDelivery is the durable Slack coordinate and delivery mode one +// completed Deliver call reports. +// +// NotificationDelivery, not Delivery: situation.Delivery is Plan 2's alert +// delivery type in snapshot.go (R7). +type NotificationDelivery struct { + Channel string + MessageTS string + DeliveredAs string // root | thread | broadcast | delayed_thread | system +} + +// SlackDeliveryState is the bounded installation-level Slack delivery health +// snapshot the worker reads once per round to decide whether to probe, open +// a gap, or reactivate configuration-blocked work. +type SlackDeliveryState struct { + // FirstFailureAt anchors the current CONTINUOUS failure window. Nil + // means Slack delivery is currently healthy. It is set by the first + // retryable/configuration failure and cleared by any success — it never + // slides forward while failures continue. + FirstFailureAt *time.Time + LastSuccessAt *time.Time + // OpenGapGeneration is the generation currently open or replaying, nil + // when none is. OpenGapStatus is that generation's status ("open" or + // "replaying"), empty when there is none. + OpenGapGeneration *string + OpenGapStatus string + // ConfigurationGeneration is the durable Slack-configuration generation. + // Startup with corrected configuration increments it and returns + // blocked intents to pending. + ConfigurationGeneration int64 + // BlockedConfigurationCount is how many intents are currently held in + // blocked_configuration. + BlockedConfigurationCount int + LastWarningAt *time.Time + UpdatedAt time.Time +} + +// NotificationStore is the durable notification-intent and Delivery-gap +// surface the worker drives. *store.Store implements it. +// +// The first fourteen methods are the plan's Cross-Task contract verbatim. +// GetSlackDeliveryState and CompleteDeliveryGap are additive: the worker +// cannot decide "probe while a failure window or gap exists" or "mark the +// generation complete only when no replayable intent remains" without +// them, and both are bounded reads/writes over the same two tables. +// +//nolint:interfacebloat // one durable ledger's whole lifecycle, deliberately: splitting it would let a partial implementation claim work it cannot acknowledge. +type NotificationStore interface { + RecoverExpiredNotificationClaims(ctx context.Context, now time.Time) (int, error) + HeartbeatNotificationClaim(ctx context.Context, claim NotificationClaim, now time.Time, lease time.Duration) error + ReleaseNotificationClaim(ctx context.Context, claim NotificationClaim, now time.Time) error + ReactivateConfigurationBlocked(ctx context.Context, configurationGeneration int64, now time.Time) (int, error) + RedriveFailedNotificationIntent(ctx context.Context, intentID string, now time.Time) error + ObserveSlackFailure(ctx context.Context, errorClass string, now time.Time) error + ObserveSlackSuccess(ctx context.Context, now time.Time) error + OpenDueDeliveryGap(ctx context.Context, now time.Time, threshold time.Duration) (bool, error) + RecoverDeliveryGap(ctx context.Context, now time.Time) (string, bool, error) + ClaimNotificationIntents(ctx context.Context, owner string, now time.Time, lease time.Duration, limit int) ([]NotificationClaim, error) + MarkNotificationDelivered(ctx context.Context, claim NotificationClaim, delivery NotificationDelivery, now time.Time) error + RetryNotificationIntent(ctx context.Context, claim NotificationClaim, errorClass string, retryAt time.Time) error + BlockNotificationConfiguration(ctx context.Context, claim NotificationClaim, errorClass string, now time.Time) error + FailNotificationIntent(ctx context.Context, claim NotificationClaim, errorClass string, now time.Time) error + + GetSlackDeliveryState(ctx context.Context) (SlackDeliveryState, error) + CompleteDeliveryGap(ctx context.Context, now time.Time) (string, bool, error) +} + +// NotificationDeliverer renders and sends exactly one Slack call per intent +// and reports Slack readiness. cmd/alertint's SituationDeliverer (Task 6) +// implements it; it writes no Store state of its own. +type NotificationDeliverer interface { + Probe(ctx context.Context) error + Deliver(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) +} + +// ---------------------------------------------------------------------- +// Failure classification. +// ---------------------------------------------------------------------- + +// DeliveryErrorClass is the closed classification the worker resolves every +// failed Deliver/Probe call into. It mirrors the Slack client's own +// classification (internal/notify/slack.ErrorClass) without this package +// depending on it: the deliverer adapter, which already owns the Slack +// wire, translates. +type DeliveryErrorClass string + +const ( + // DeliveryRetryable covers transport failures, 5xx, rate limiting, and + // every uncertain outcome. It retries indefinitely. + DeliveryRetryable DeliveryErrorClass = "retryable" + // DeliveryConfigurationBlocking covers a definite token/scope/channel/ + // authentication rejection. It is durable, keeps its attempts, and + // waits for a corrected configuration generation — never exhausted. + DeliveryConfigurationBlocking DeliveryErrorClass = "configuration_blocking" + // DeliveryInvalid covers an invalid durable intent or another + // non-recoverable programming/data error this build proved. It is the + // only class that becomes `failed`, and even that is redriveable. + DeliveryInvalid DeliveryErrorClass = "invalid" +) + +// DeliveryFailure is the classification a deliverer error may carry. An +// error that does not implement it is treated as retryable: this worker +// never dead-letters durable operator history on an error it cannot prove +// is permanent. +type DeliveryFailure interface { + error + // DeliveryErrorClass reports which closed outcome this failure is. + DeliveryErrorClass() DeliveryErrorClass + // DeliveryErrorCode is the bounded, lowercase-identifier error class + // recorded on the intent (never raw error text or a provider body). + DeliveryErrorCode() string + // DeliveryRetryAfter is the provider-requested wait, or zero. + DeliveryRetryAfter() time.Duration +} + +// classifyDeliveryFailure resolves err into (class, bounded code, retry +// hint). An unclassified error is retryable with the generic code +// "delivery_failed" — nothing that cannot prove itself permanent may ever +// close a delivery obligation. +func classifyDeliveryFailure(err error) (DeliveryErrorClass, string, time.Duration) { + var failure DeliveryFailure + if errors.As(err, &failure) { + code := boundedErrorCode(failure.DeliveryErrorCode()) + switch failure.DeliveryErrorClass() { + case DeliveryConfigurationBlocking: + return DeliveryConfigurationBlocking, code, 0 + case DeliveryInvalid: + return DeliveryInvalid, code, 0 + case DeliveryRetryable: + return DeliveryRetryable, code, failure.DeliveryRetryAfter() + default: + // An unknown class is not proof of a permanent condition. + return DeliveryRetryable, code, failure.DeliveryRetryAfter() + } + } + return DeliveryRetryable, defaultDeliveryErrorCode, 0 +} + +// defaultDeliveryErrorCode is what an unclassified or unusable error code +// records as. +const defaultDeliveryErrorCode = "delivery_failed" + +// maxDeliveryErrorCode mirrors the store's own last_error_class bound. +const maxDeliveryErrorCode = 64 + +// boundedErrorCode coerces a deliverer's error code into the closed +// lowercase-identifier shape the durable last_error_class column accepts, +// so no deliverer — including a third-party one — can ever make an +// acknowledgement unwritable, and no raw error text can reach the column. +func boundedErrorCode(code string) string { + out := make([]rune, 0, len(code)) + for _, r := range strings.ToLower(strings.TrimSpace(code)) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + out = append(out, r) + default: + out = append(out, '_') + } + if len(out) == maxDeliveryErrorCode { + break + } + } + if len(out) == 0 || out[0] < 'a' || out[0] > 'z' { + return defaultDeliveryErrorCode + } + return string(out) +} + +// ---------------------------------------------------------------------- +// Deterministic gap identities. +// ---------------------------------------------------------------------- + +// NewGapGenerationID derives the stable identity of the Delivery-gap +// generation opening over the continuous failure window that began at +// firstFailureAt. Deriving it from the window's own anchor (rather than a +// fresh random ID) makes gap opening idempotent: two racing openers compute +// the same primary key, so exactly one row can exist per outage window. +func NewGapGenerationID(firstFailureAt time.Time) string { + return intentIdentity("slack_delivery_gap:" + firstFailureAt.UTC().Format(time.RFC3339Nano)) +} + +// GapRecoveryIntent builds the one bounded ADR-0042 System notice that +// precedes gapGeneration's backlog replay. Its identity is derived from the +// generation, so recovering the same generation twice can only ever produce +// the same row. +func GapRecoveryIntent(gapGeneration string, now time.Time) (model.NotificationIntent, error) { + key := boundedText("installation_gap_recovery:"+gapGeneration, maxHistoryIdentifier) + generation := gapGeneration + intent := model.NotificationIntent{ + ID: intentIdentity("intent:" + key), + IdempotencyKey: key, + EffectClass: model.EffectInstallationGapRecovery, + GapGeneration: &generation, + ClientMessageID: intentIdentity("client_message:" + key), + Status: model.IntentPending, + CreatedAt: now.UTC(), + } + if err := intent.Validate(); err != nil { + return model.NotificationIntent{}, fmt.Errorf("situation: gap recovery intent: %w", err) + } + return intent, nil +} + +// ---------------------------------------------------------------------- +// Worker configuration. +// ---------------------------------------------------------------------- + +const ( + defaultNotificationPoll = time.Second + defaultNotificationLease = 300 * time.Second + defaultNotificationHeartbeat = 30 * time.Second + defaultNotificationBatch = 25 + defaultNotificationRetryInitial = 5 * time.Second + defaultNotificationRetryMax = 300 * time.Second + defaultNotificationJitter = 0.2 + // defaultDeliveryGapThreshold is the spec's fixed five continuous + // minutes of failure before a durable Delivery gap opens. It is a + // constant of the protocol, not an operator knob; the field exists so + // tests can compress it. + defaultDeliveryGapThreshold = 5 * time.Minute + // notificationWarnCadence paces the bounded retry WARNs while an + // outage continues, so an hour-long Slack outage costs a handful of + // lines rather than one per attempt. + notificationWarnCadence = time.Minute +) + +// NotificationWorkerConfig controls the worker's lease fencing, poll +// cadence, batch size, retry schedule, and gap threshold. It deliberately +// has NO maximum-attempts field: valid Slack effects retry indefinitely. +type NotificationWorkerConfig struct { + // Owner identifies this worker instance to the store's lease fencing. + // Required — there is no default. + Owner string + // Poll is how often the background loop wakes on its own. Default 1s. + Poll time.Duration + // Lease is how long a claimed intent is held before another worker (or + // this worker's own recovery sweep) may reclaim it. Default 300s. + Lease time.Duration + // Heartbeat is how often an in-flight claim's lease is renewed. + // Default 30s, well under Lease. + Heartbeat time.Duration + // Batch bounds how many intents one round claims. Default 25. + Batch int + // RetryInitial and RetryMax bound the exponential retry schedule. + // Defaults 5s and 300s. + RetryInitial time.Duration + RetryMax time.Duration + // JitterFraction spreads each retry by +/- this fraction. Default 0.2. + JitterFraction float64 + // GapThreshold is how long failures must stay continuous before a + // durable Delivery gap opens. Default 5m. + GapThreshold time.Duration + // Rand is the [0,1) source the retry jitter reads. Default rand.Float64. + Rand func() float64 +} + +func (c NotificationWorkerConfig) withDefaults() NotificationWorkerConfig { + if c.Poll <= 0 { + c.Poll = defaultNotificationPoll + } + if c.Lease <= 0 { + c.Lease = defaultNotificationLease + } + if c.Heartbeat <= 0 { + c.Heartbeat = defaultNotificationHeartbeat + } + if c.Batch <= 0 { + c.Batch = defaultNotificationBatch + } + if c.RetryInitial <= 0 { + c.RetryInitial = defaultNotificationRetryInitial + } + if c.RetryMax <= 0 { + c.RetryMax = defaultNotificationRetryMax + } + if c.JitterFraction < 0 || c.JitterFraction >= 1 { + c.JitterFraction = defaultNotificationJitter + } + if c.GapThreshold <= 0 { + c.GapThreshold = defaultDeliveryGapThreshold + } + if c.Rand == nil { + c.Rand = rand.Float64 + } + return c +} + +// NotificationWorkerStats is the bounded counter set the worker exposes for +// logs and (Task 9) MCP delivery-state fields. Plan 3 adds no OTel metric +// instruments (R8), so these are the operational signal. +type NotificationWorkerStats struct { + Claimed int64 + Delivered int64 + Retried int64 + Blocked int64 + Failed int64 + Superseded int64 + ClaimsLost int64 + Probes int64 + ProbeFailures int64 + GapsOpened int64 + GapsRecovered int64 + GapsCompleted int64 + Reactivated int64 +} + +// NotificationWorker polls the durable notification-intent ledger, claims +// due intents under a fenced lease, delivers each one exactly once per +// claim, and drives the Delivery-gap state machine. +// +// It is safe for exactly one Start/Stop lifecycle; RunOnce may additionally +// be called directly (tests, or a one-shot drain) without ever calling +// Start. +type NotificationWorker struct { + store NotificationStore + deliverer NotificationDeliverer + cfg NotificationWorkerConfig + now Clock + logger *slog.Logger + + wakeCh chan struct{} + stopCh chan struct{} + doneCh chan struct{} + + startOnce sync.Once + stopOnce sync.Once + started atomic.Bool + + mu sync.Mutex + // inflight holds every claim this worker currently owns, so Stop can + // release whatever the final pass was still holding (R6). + inflight map[string]NotificationClaim + // probeFailures drives the probe's own backoff, and lastProbeAt/ + // lastWarnAt pace probing and retry WARNs. All worker-local: the + // durable record of the outage is slack_delivery_state. + probeFailures int + lastProbeAt time.Time + lastWarnAt time.Time + probedOnce bool + + stats NotificationWorkerStats + statsMu sync.Mutex +} + +// NewNotificationWorker creates a NotificationWorker. A nil clock falls back +// to the UTC wall clock; a nil logger falls back to slog.Default. +func NewNotificationWorker(store NotificationStore, deliverer NotificationDeliverer, + cfg NotificationWorkerConfig, clock Clock, logger *slog.Logger) *NotificationWorker { + if clock == nil { + clock = func() time.Time { return time.Now().UTC() } + } + if logger == nil { + logger = slog.Default() + } + return &NotificationWorker{ + store: store, + deliverer: deliverer, + cfg: cfg.withDefaults(), + now: clock, + logger: logger, + wakeCh: make(chan struct{}, 1), + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + inflight: map[string]NotificationClaim{}, + } +} + +// Stats returns a snapshot of the worker's bounded counters. +func (w *NotificationWorker) Stats() NotificationWorkerStats { + w.statsMu.Lock() + defer w.statsMu.Unlock() + return w.stats +} + +func (w *NotificationWorker) count(f func(*NotificationWorkerStats)) { + w.statsMu.Lock() + defer w.statsMu.Unlock() + f(&w.stats) +} + +// RunOnce runs one full delivery round: recover abandoned claims, advance +// the Delivery-gap state machine, then claim and deliver up to cfg.Batch +// intents sequentially. It returns how many intents it actually delivered +// an acknowledgement for. +func (w *NotificationWorker) RunOnce(ctx context.Context) (int, error) { + now := w.now().UTC() + + if n, err := w.store.RecoverExpiredNotificationClaims(ctx, now); err != nil { + w.logger.Error("situation: notification worker: recover expired claims failed", "err", err) + } else if n > 0 { + w.logger.Info("situation: notification worker: recovered abandoned claims", "count", n) + } + + state, err := w.store.GetSlackDeliveryState(ctx) + if err != nil { + return 0, fmt.Errorf("situation: notification worker: read slack delivery state: %w", err) + } + w.advanceGapState(ctx, state, now) + + claims, err := w.store.ClaimNotificationIntents(ctx, w.cfg.Owner, now, w.cfg.Lease, w.cfg.Batch) + if err != nil { + return 0, fmt.Errorf("situation: notification worker: claim notification intents: %w", err) + } + handled := 0 + for _, claim := range claims { + if err := ctx.Err(); err != nil { + w.releaseClaim(claim) //nolint:contextcheck // by design: releaseClaim uses its own detachedWriteContext; ctx is already done here + return handled, err + } + w.count(func(s *NotificationWorkerStats) { s.Claimed++ }) + w.processOne(ctx, claim) + handled++ + } + return handled, nil +} + +// advanceGapState drives the whole gap lifecycle for one round: probe while +// a failure window or gap exists, open a generation once failures have been +// continuous for the threshold, recover it once Slack answers again, and +// complete it once nothing replayable remains. +func (w *NotificationWorker) advanceGapState(ctx context.Context, state SlackDeliveryState, now time.Time) { + if w.shouldProbe(state, now) { + w.probe(ctx, state, now) + } + if state.FirstFailureAt != nil { + opened, err := w.store.OpenDueDeliveryGap(ctx, now, w.cfg.GapThreshold) + if err != nil { + w.logger.Error("situation: notification worker: open delivery gap failed", "err", err) + } else if opened { + w.count(func(s *NotificationWorkerStats) { s.GapsOpened++ }) + w.logger.Warn("situation: notification worker: slack delivery gap opened", + "first_failure_at", state.FirstFailureAt.Format(time.RFC3339), + "continuous_for", now.Sub(*state.FirstFailureAt).String()) + } + } + // Bounded: at most one completion per round keeps this cheap, and a + // second finished generation completes on the next tick. + if generation, done, err := w.store.CompleteDeliveryGap(ctx, now); err != nil { + w.logger.Error("situation: notification worker: complete delivery gap failed", "err", err) + } else if done { + w.count(func(s *NotificationWorkerStats) { s.GapsCompleted++ }) + w.logger.Info("situation: notification worker: slack delivery gap replay complete", "gap_generation", generation) + } +} + +// shouldProbe answers the spec's "probe while configuration is blocked or a +// failure window/gap exists", plus exactly one probe on the worker's first +// round so corrected startup configuration is noticed without waiting for a +// delivery to fail again. Probes back off with the same exponential +// schedule retries use, so an hour-long outage costs a handful of calls. +func (w *NotificationWorker) shouldProbe(state SlackDeliveryState, now time.Time) bool { + w.mu.Lock() + defer w.mu.Unlock() + if !w.probedOnce { + return true + } + // A REPLAYING generation is a recovered one: its own deliveries prove + // Slack health, so only an outage window, a still-open generation, or + // durably blocked configuration keeps probing. + if state.FirstFailureAt == nil && state.OpenGapStatus != "open" && state.BlockedConfigurationCount == 0 { + return false + } + wait := notificationRetryDelay(w.probeFailures, 0, w.cfg.RetryInitial, w.cfg.RetryMax, 0, 0) + return !now.Before(w.lastProbeAt.Add(wait)) +} + +// probe asks the deliverer whether Slack is usable and applies the outcome: +// a success clears the failure window, reactivates configuration-blocked +// intents, and moves any open gap into replay; a failure keeps the window +// continuous. +func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState, now time.Time) { + w.mu.Lock() + w.probedOnce = true + w.lastProbeAt = now + w.mu.Unlock() + w.count(func(s *NotificationWorkerStats) { s.Probes++ }) + + if err := w.deliverer.Probe(ctx); err != nil { + w.mu.Lock() + w.probeFailures++ + w.mu.Unlock() + w.count(func(s *NotificationWorkerStats) { s.ProbeFailures++ }) + class, code, _ := classifyDeliveryFailure(err) + if class != DeliveryInvalid { + w.observeFailure(ctx, state, code, now) + } + return + } + + w.mu.Lock() + w.probeFailures = 0 + w.mu.Unlock() + if err := w.store.ObserveSlackSuccess(ctx, now); err != nil { + w.logger.Error("situation: notification worker: record slack success failed", "err", err) + } + if state.BlockedConfigurationCount > 0 { + n, err := w.store.ReactivateConfigurationBlocked(ctx, state.ConfigurationGeneration+1, now) + if err != nil { + w.logger.Error("situation: notification worker: reactivate configuration-blocked intents failed", "err", err) + } else if n > 0 { + w.count(func(s *NotificationWorkerStats) { s.Reactivated += int64(n) }) + w.logger.Info("situation: notification worker: slack configuration corrected; reactivated blocked intents", + "count", n, "configuration_generation", state.ConfigurationGeneration+1) + } + } + if generation, recovered, err := w.store.RecoverDeliveryGap(ctx, now); err != nil { + w.logger.Error("situation: notification worker: recover delivery gap failed", "err", err) + } else if recovered { + w.count(func(s *NotificationWorkerStats) { s.GapsRecovered++ }) + w.logger.Warn("situation: notification worker: slack delivery recovered; replaying gap", + "gap_generation", generation) + } +} + +// observeFailure records one Slack failure against the continuous window and +// emits the bounded WARNs the console action trail expects: one on the first +// failure of a window, then paced retry WARNs — never one per attempt. +func (w *NotificationWorker) observeFailure(ctx context.Context, state SlackDeliveryState, code string, now time.Time) { + if err := w.store.ObserveSlackFailure(ctx, code, now); err != nil { + w.logger.Error("situation: notification worker: record slack failure failed", "err", err) + return + } + if state.FirstFailureAt == nil { + w.mu.Lock() + w.lastWarnAt = now + w.mu.Unlock() + w.logger.Warn("situation: notification worker: slack delivery failing; effects are delayed", + "error_class", code) + return + } + w.mu.Lock() + due := now.Sub(w.lastWarnAt) >= notificationWarnCadence + if due { + w.lastWarnAt = now + } + w.mu.Unlock() + if due { + w.logger.Warn("situation: notification worker: slack delivery still failing", + "error_class", code, "continuous_for", now.Sub(*state.FirstFailureAt).String()) + } +} + +// processOne delivers exactly one claimed intent and acknowledges the real +// outcome. The Slack call happens outside every database transaction, under +// a heartbeat that abandons the attempt if the lease moves on. +func (w *NotificationWorker) processOne(ctx context.Context, claim NotificationClaim) { + w.trackClaim(claim) + defer w.untrackClaim(claim) + + deliverCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var leaseLost atomic.Bool + hbDone := make(chan struct{}) + go w.heartbeatLoop(deliverCtx, cancel, claim, &leaseLost, hbDone) + + delivery, deliverErr := w.deliverer.Deliver(deliverCtx, claim.Intent) + + cancel() + <-hbDone + + if leaseLost.Load() { + // The lease moved on mid-flight (a sweep reclaimed it, or a + // concurrent commit superseded this root projection). Acknowledging + // now would race whatever owns the row; the durable outcome is + // whatever that owner writes. + w.count(func(s *NotificationWorkerStats) { s.ClaimsLost++ }) + w.logger.Warn("situation: notification worker: lease lost mid-delivery; abandoning claim", + "intent_id", claim.Intent.ID, "effect_class", string(claim.Intent.EffectClass)) + return + } + + // The acknowledgement must land even when the delivery context was + // canceled, or a completed Slack call would be replayed forever. + writeCtx, writeCancel := detachedWriteContext() + defer writeCancel() + now := w.now().UTC() + + if deliverErr == nil { + w.acknowledgeDelivered(writeCtx, claim, delivery, now) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context + return + } + w.acknowledgeFailure(writeCtx, claim, deliverErr, now) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context +} + +func (w *NotificationWorker) acknowledgeDelivered(ctx context.Context, claim NotificationClaim, + delivery NotificationDelivery, now time.Time) { + // Slack answered, so the dependency is healthy regardless of whether + // this particular intent's row was still ours to write. + if err := w.store.ObserveSlackSuccess(ctx, now); err != nil { + w.logger.Error("situation: notification worker: record slack success failed", "err", err) + } + err := w.store.MarkNotificationDelivered(ctx, claim, delivery, now) + switch { + case err == nil: + w.count(func(s *NotificationWorkerStats) { s.Delivered++ }) + case errors.Is(err, ErrNotificationIntentSuperseded): + // R4: a newer root projection replaced this one mid-flight. The + // message that just went out is the older projection's; the newer + // one edits the same root next round. Expected, not a failure. + w.count(func(s *NotificationWorkerStats) { s.Superseded++ }) + w.logger.Info("situation: notification worker: root projection superseded mid-delivery", + "intent_id", claim.Intent.ID, "situation_id", derefString(claim.Intent.SituationID)) + case errors.Is(err, ErrNotificationClaimLost): + w.count(func(s *NotificationWorkerStats) { s.ClaimsLost++ }) + w.logger.Warn("situation: notification worker: delivered acknowledgement lost its claim", + "intent_id", claim.Intent.ID) + default: + w.logger.Error("situation: notification worker: acknowledge delivery failed", + "intent_id", claim.Intent.ID, "err", err) + } +} + +func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim NotificationClaim, deliverErr error, now time.Time) { + class, code, retryAfter := classifyDeliveryFailure(deliverErr) + + state, stateErr := w.store.GetSlackDeliveryState(ctx) + if stateErr != nil { + w.logger.Error("situation: notification worker: read slack delivery state failed", "err", stateErr) + } + if class != DeliveryInvalid { + // An invalid payload is this build's own bug, not a Slack outage: + // it must never open a Delivery gap. + w.observeFailure(ctx, state, code, now) + } + + var ackErr error + switch class { + case DeliveryRetryable: + ackErr = w.retryClaim(ctx, claim, code, retryAfter, now) + case DeliveryConfigurationBlocking: + ackErr = w.store.BlockNotificationConfiguration(ctx, claim, code, now) + if ackErr == nil { + w.count(func(s *NotificationWorkerStats) { s.Blocked++ }) + w.logger.Warn("situation: notification worker: slack configuration rejected the effect; blocking until corrected", + "intent_id", claim.Intent.ID, "error_class", code) + } + case DeliveryInvalid: + ackErr = w.store.FailNotificationIntent(ctx, claim, code, now) + if ackErr == nil { + w.count(func(s *NotificationWorkerStats) { s.Failed++ }) + w.logger.Error("situation: notification worker: invalid durable intent; failed pending operator redrive", + "intent_id", claim.Intent.ID, "error_class", code) + } + default: + ackErr = w.retryClaim(ctx, claim, code, retryAfter, now) + } + switch { + case ackErr == nil: + case errors.Is(ackErr, ErrNotificationIntentSuperseded), errors.Is(ackErr, ErrNotificationClaimLost): + w.count(func(s *NotificationWorkerStats) { s.ClaimsLost++ }) + default: + w.logger.Error("situation: notification worker: acknowledge failure outcome failed", + "intent_id", claim.Intent.ID, "err", ackErr) + } +} + +// retryClaim schedules the next indefinite retry for one failed claim. +func (w *NotificationWorker) retryClaim(ctx context.Context, claim NotificationClaim, code string, + retryAfter time.Duration, now time.Time) error { + delay := notificationRetryDelay(claim.Intent.AttemptCount, retryAfter, + w.cfg.RetryInitial, w.cfg.RetryMax, w.cfg.JitterFraction, w.cfg.Rand()) + err := w.store.RetryNotificationIntent(ctx, claim, code, now.Add(delay)) + if err == nil { + w.count(func(s *NotificationWorkerStats) { s.Retried++ }) + } + return err +} + +// heartbeatLoop renews claim's lease until ctx is done. A failed renewal +// marks the lease lost and cancels the in-flight Slack call rather than +// letting it complete under a lease this worker no longer holds. +func (w *NotificationWorker) heartbeatLoop(ctx context.Context, cancel context.CancelFunc, + claim NotificationClaim, leaseLost *atomic.Bool, done chan<- struct{}) { + defer close(done) + + ticker := time.NewTicker(w.cfg.Heartbeat) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + beatCtx, beatCancel := detachedWriteContext() + err := w.store.HeartbeatNotificationClaim(beatCtx, claim, w.now().UTC(), w.cfg.Lease) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context + beatCancel() + if err != nil { + w.logger.Warn("situation: notification worker: heartbeat failed; abandoning claim", + "intent_id", claim.Intent.ID, "err", err) + leaseLost.Store(true) + cancel() + return + } + } + } +} + +func (w *NotificationWorker) trackClaim(claim NotificationClaim) { + w.mu.Lock() + defer w.mu.Unlock() + w.inflight[claim.Intent.ID] = claim +} + +func (w *NotificationWorker) untrackClaim(claim NotificationClaim) { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.inflight, claim.Intent.ID) +} + +// releaseClaim hands one still-held claim straight back, so a shutdown never +// leaves a durable obligation waiting out a full lease. It deliberately +// takes no caller context: it runs on the shutdown/cancellation path, where +// the caller's context is usually already done, and a release that gets +// canceled would leave the intent waiting out its whole lease. +func (w *NotificationWorker) releaseClaim(claim NotificationClaim) { + ctx, cancel := detachedWriteContext() + defer cancel() + if err := w.store.ReleaseNotificationClaim(ctx, claim, w.now().UTC()); err != nil && + !errors.Is(err, ErrNotificationClaimLost) && !errors.Is(err, ErrNotificationIntentSuperseded) { + w.logger.Warn("situation: notification worker: release claim failed", + "intent_id", claim.Intent.ID, "err", err) + } + w.untrackClaim(claim) +} + +// Start launches the background loop and returns immediately. Safe to call +// at most once; later calls are no-ops. +func (w *NotificationWorker) Start(ctx context.Context) { + w.startOnce.Do(func() { + w.started.Store(true) + go w.run(ctx) + }) +} + +func (w *NotificationWorker) run(ctx context.Context) { + defer close(w.doneCh) + + ticker := time.NewTicker(w.cfg.Poll) + defer ticker.Stop() + + for { + if _, err := w.RunOnce(ctx); err != nil && + !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + w.logger.Error("situation: notification worker: round failed", "err", err) + } + + select { + case <-ctx.Done(): + return + case <-w.stopCh: + return + case <-ticker.C: + case <-w.wakeCh: + } + } +} + +// Wake nudges the background loop to run another round immediately. Never +// blocks; a coalesced Wake is harmless because due rows poll again anyway. +func (w *NotificationWorker) Wake() { + select { + case w.wakeCh <- struct{}{}: + default: + } +} + +// Stop ends the background loop, then runs ONE bounded final pass under ctx +// and releases every claim still held (R6). The notification worker never +// joins Plan 2's shutdown drain rounds: an external Slack outage must never +// hold shutdown open, and committed intents left pending are simply +// reclaimed at the next startup. +// +// Stopping a worker that was never started still runs the final pass and +// the release, rather than waiting out ctx for a loop that does not exist. +// Stop is idempotent. +func (w *NotificationWorker) Stop(ctx context.Context) error { + w.stopOnce.Do(func() { close(w.stopCh) }) + + var loopErr error + if w.started.Load() { + select { + case <-w.doneCh: + case <-ctx.Done(): + loopErr = ctx.Err() + } + } + + if loopErr == nil { + if _, err := w.RunOnce(ctx); err != nil && + !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + w.logger.Error("situation: notification worker: final pass failed", "err", err) + } + } + for _, claim := range w.heldClaims() { + w.releaseClaim(claim) //nolint:contextcheck // by design: releaseClaim uses its own detachedWriteContext, so shutdown still releases when ctx is done + } + return loopErr +} + +func (w *NotificationWorker) heldClaims() []NotificationClaim { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]NotificationClaim, 0, len(w.inflight)) + for _, claim := range w.inflight { + out = append(out, claim) + } + return out +} + +// notificationRetryDelay is the indefinite retry schedule: exponential from +// initial, doubling per consumed attempt, capped at maxDelay, then spread by +// +/- jitter, and finally overridden by a LONGER provider-requested +// Retry-After. It has no terminal value at any attempt count — attempt 1, +// 100, and 100000 all return a finite, bounded delay. +func notificationRetryDelay(attempt int, retryAfter, initial, maxDelay time.Duration, + jitter, rnd float64) time.Duration { + if attempt < 1 { + attempt = 1 + } + delay := maxDelay + if shift := attempt - 1; shift < 32 { + if scaled := initial << uint(shift); scaled > 0 && scaled < maxDelay { // #nosec G115 -- shift < 32 checked immediately above + delay = scaled + } + } + if jitter > 0 { + delay = time.Duration(float64(delay) * (1 + jitter*(2*rnd-1))) + } + if delay < time.Duration(0) { + delay = initial + } + if retryAfter > delay { + delay = retryAfter + } + return delay +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/internal/situation/notification_worker_test.go b/internal/situation/notification_worker_test.go new file mode 100644 index 0000000..8b73049 --- /dev/null +++ b/internal/situation/notification_worker_test.go @@ -0,0 +1,749 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package situation + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 7 worker fixtures. +// ---------------------------------------------------------------------- + +func nwLogger() *slog.Logger { + return slog.New(slog.DiscardHandler) +} + +// nwDeliveryError is a deliverer error carrying the closed classification the +// worker resolves outcomes with, exactly as cmd/alertint's Slack adapter +// does for a *slack.APIError. +type nwDeliveryError struct { + class DeliveryErrorClass + code string + retryAfter time.Duration +} + +func (f nwDeliveryError) Error() string { return fmt.Sprintf("slack: %s: %s", f.class, f.code) } +func (f nwDeliveryError) DeliveryErrorClass() DeliveryErrorClass { return f.class } +func (f nwDeliveryError) DeliveryErrorCode() string { return f.code } +func (f nwDeliveryError) DeliveryRetryAfter() time.Duration { return f.retryAfter } + +type nwRetry struct { + intentID string + class string + retryAt time.Time +} + +// nwStore is a race-safe in-memory NotificationStore recording every call +// the worker makes. +type nwStore struct { + mu sync.Mutex + + state SlackDeliveryState + batches [][]NotificationClaim + deliveredAs []NotificationDelivery + delivered []string + retried []nwRetry + blocked []string + failed []string + released []string + heartbeats int + successes int + failures []string + gapOpens []time.Duration + completes int + recovers int + reactivated []int64 + recovered int + + deliverAckErr error + heartbeatErr error + openGap bool + recoverGap string + recoverOK bool + completeGap string + completeOK bool +} + +func (s *nwStore) RecoverExpiredNotificationClaims(_ context.Context, _ time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.recovered++ + return 0, nil +} + +func (s *nwStore) HeartbeatNotificationClaim(_ context.Context, _ NotificationClaim, _ time.Time, _ time.Duration) error { + s.mu.Lock() + defer s.mu.Unlock() + s.heartbeats++ + return s.heartbeatErr +} + +func (s *nwStore) ReleaseNotificationClaim(_ context.Context, claim NotificationClaim, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.released = append(s.released, claim.Intent.ID) + return nil +} + +func (s *nwStore) ReactivateConfigurationBlocked(_ context.Context, generation int64, _ time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.reactivated = append(s.reactivated, generation) + return 1, nil +} + +func (s *nwStore) RedriveFailedNotificationIntent(_ context.Context, _ string, _ time.Time) error { + return nil +} + +func (s *nwStore) ObserveSlackFailure(_ context.Context, class string, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.failures = append(s.failures, class) + return nil +} + +func (s *nwStore) ObserveSlackSuccess(_ context.Context, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.successes++ + return nil +} + +func (s *nwStore) OpenDueDeliveryGap(_ context.Context, _ time.Time, threshold time.Duration) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.gapOpens = append(s.gapOpens, threshold) + return s.openGap, nil +} + +func (s *nwStore) RecoverDeliveryGap(_ context.Context, _ time.Time) (string, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.recovers++ + return s.recoverGap, s.recoverOK, nil +} + +func (s *nwStore) CompleteDeliveryGap(_ context.Context, _ time.Time) (string, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.completes++ + return s.completeGap, s.completeOK, nil +} + +func (s *nwStore) GetSlackDeliveryState(context.Context) (SlackDeliveryState, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.state, nil +} + +func (s *nwStore) ClaimNotificationIntents(_ context.Context, _ string, _ time.Time, _ time.Duration, _ int) ([]NotificationClaim, error) { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.batches) == 0 { + return nil, nil + } + batch := s.batches[0] + s.batches = s.batches[1:] + return batch, nil +} + +func (s *nwStore) MarkNotificationDelivered(_ context.Context, claim NotificationClaim, + delivery NotificationDelivery, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.deliverAckErr != nil { + return s.deliverAckErr + } + s.delivered = append(s.delivered, claim.Intent.ID) + s.deliveredAs = append(s.deliveredAs, delivery) + return nil +} + +func (s *nwStore) RetryNotificationIntent(_ context.Context, claim NotificationClaim, class string, retryAt time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.retried = append(s.retried, nwRetry{intentID: claim.Intent.ID, class: class, retryAt: retryAt}) + return nil +} + +func (s *nwStore) BlockNotificationConfiguration(_ context.Context, claim NotificationClaim, class string, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.blocked = append(s.blocked, claim.Intent.ID+":"+class) + return nil +} + +func (s *nwStore) FailNotificationIntent(_ context.Context, claim NotificationClaim, class string, _ time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.failed = append(s.failed, claim.Intent.ID+":"+class) + return nil +} + +func (s *nwStore) snapshot(read func(*nwStore)) { + s.mu.Lock() + defer s.mu.Unlock() + read(s) +} + +// nwDeliverer is a race-safe fake NotificationDeliverer. +type nwDeliverer struct { + mu sync.Mutex + probeErr error + probeCalls int + calls []model.NotificationIntent + deliver func(model.NotificationIntent) (NotificationDelivery, error) +} + +func (d *nwDeliverer) Probe(context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + d.probeCalls++ + return d.probeErr +} + +func (d *nwDeliverer) Deliver(ctx context.Context, intent model.NotificationIntent) (NotificationDelivery, error) { + d.mu.Lock() + d.calls = append(d.calls, intent) + fn := d.deliver + d.mu.Unlock() + if fn == nil { + return NotificationDelivery{Channel: "C", MessageTS: "1.1", DeliveredAs: "root"}, nil + } + _ = ctx + return fn(intent) +} + +func (d *nwDeliverer) clientMessageIDs() []string { + d.mu.Lock() + defer d.mu.Unlock() + out := make([]string, 0, len(d.calls)) + for _, c := range d.calls { + out = append(out, c.ClientMessageID) + } + return out +} + +func nwClaim(id string, attempt int) NotificationClaim { + situationID := "sit-1" + transitionID := "tr-1" + sequence := 1 + version := 1 + return NotificationClaim{ + Intent: model.NotificationIntent{ + ID: id, + IdempotencyKey: "key:" + id, + EffectClass: model.EffectRootSync, + SituationID: &situationID, + TransitionID: &transitionID, + TransitionSequence: &sequence, + SummaryVersion: &version, + ClientMessageID: "client:" + id, + Status: model.IntentPending, + AttemptCount: attempt, + CreatedAt: time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC), + }, + ClaimOwner: "notify-a", + ClaimToken: 1, + } +} + +// nwClock is a mutable test clock safe to advance while worker goroutines +// read it. +type nwClock struct { + mu sync.Mutex + at time.Time +} + +func (c *nwClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.at +} + +func (c *nwClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.at = c.at.Add(d) +} + +// nwWorker builds a worker on a fixed clock with a deterministic jitter +// source (rnd = 0.5 => no jitter) and a fast heartbeat. +func nwWorker(store NotificationStore, deliverer NotificationDeliverer, now time.Time) *NotificationWorker { + return NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", + Heartbeat: time.Hour, + Rand: func() float64 { return 0.5 }, + }, func() time.Time { return now }, nwLogger()) +} + +// ---------------------------------------------------------------------- +// Step 4: indefinite retry. +// ---------------------------------------------------------------------- + +// TestNotificationWorkerRetriesIndefinitely proves the retry schedule has no +// terminal value at ANY attempt count: attempt 1, 100, and 100000 all +// return a finite delay bounded by five minutes plus its jitter. +func TestNotificationWorkerRetriesIndefinitely(t *testing.T) { + initial, maxDelay := 5*time.Second, 5*time.Minute + for _, attempt := range []int{1, 2, 100, 100000} { + for _, rnd := range []float64{0, 0.5, 1} { + delay := notificationRetryDelay(attempt, 0, initial, maxDelay, 0.2, rnd) + if delay <= 0 { + t.Fatalf("attempt %d rnd %v: delay %s, want a positive retry delay", attempt, rnd, delay) + } + if delay > time.Duration(float64(maxDelay)*1.2) { + t.Fatalf("attempt %d rnd %v: delay %s exceeds the five-minute cap plus bounded jitter", attempt, rnd, delay) + } + } + } + // The schedule really is exponential before the cap, and jittered. + if got := notificationRetryDelay(1, 0, initial, maxDelay, 0.2, 0.5); got != initial { + t.Fatalf("attempt 1 with neutral jitter = %s, want %s", got, initial) + } + if got := notificationRetryDelay(2, 0, initial, maxDelay, 0.2, 0.5); got != 2*initial { + t.Fatalf("attempt 2 with neutral jitter = %s, want %s", got, 2*initial) + } + low := notificationRetryDelay(3, 0, initial, maxDelay, 0.2, 0) + high := notificationRetryDelay(3, 0, initial, maxDelay, 0.2, 1) + if low >= high || low != time.Duration(float64(4*initial)*0.8) || high != time.Duration(float64(4*initial)*1.2) { + t.Fatalf("attempt 3 jitter spread = [%s, %s], want +/-20%% around %s", low, high, 4*initial) + } + if got := notificationRetryDelay(100, 0, initial, maxDelay, 0, 0); got != maxDelay { + t.Fatalf("attempt 100 without jitter = %s, want the %s cap", got, maxDelay) + } +} + +// TestNotificationWorkerHonorsLongerRetryAfter proves a provider-requested +// wait wins when it is longer than the computed backoff, and loses when it +// is shorter. +func TestNotificationWorkerHonorsLongerRetryAfter(t *testing.T) { + initial, maxDelay := 5*time.Second, 5*time.Minute + if got := notificationRetryDelay(1, 30*time.Minute, initial, maxDelay, 0.2, 0.5); got != 30*time.Minute { + t.Fatalf("delay with a 30m Retry-After = %s, want 30m", got) + } + if got := notificationRetryDelay(1, time.Second, initial, maxDelay, 0.2, 0.5); got != initial { + t.Fatalf("delay with a 1s Retry-After = %s, want the longer computed %s", got, initial) + } +} + +// TestNotificationWorkerNoElapsedOutageEverFailsAValidIntent drives the +// worker itself at attempts 1, 100, and 100000: every one is retried, none +// is ever failed or blocked. +func TestNotificationWorkerNoElapsedOutageEverFailsAValidIntent(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{} + for _, attempt := range []int{1, 100, 100000} { + store.batches = append(store.batches, []NotificationClaim{nwClaim(fmt.Sprintf("intent-%d", attempt), attempt)}) + } + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{}, nwDeliveryError{class: DeliveryRetryable, code: "ratelimited"} + }} + w := nwWorker(store, deliverer, now) + for i := 0; i < 3; i++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", i, err) + } + } + store.snapshot(func(s *nwStore) { + if len(s.retried) != 3 { + t.Fatalf("retried %d intents, want 3", len(s.retried)) + } + if len(s.failed) != 0 || len(s.blocked) != 0 { + t.Fatalf("failed=%v blocked=%v, want an elapsed outage to close no delivery obligation", s.failed, s.blocked) + } + for _, r := range s.retried { + if !r.retryAt.After(now) { + t.Fatalf("retry_at %s is not in the future", r.retryAt) + } + if r.retryAt.Sub(now) > time.Duration(float64(5*time.Minute)*1.2) { + t.Fatalf("retry_at %s exceeds the capped delay", r.retryAt) + } + if r.class != "ratelimited" { + t.Fatalf("recorded error class %q, want the bounded ratelimited", r.class) + } + } + if len(s.failures) != 3 { + t.Fatalf("observed %d slack failures, want one per retryable outcome", len(s.failures)) + } + }) +} + +// TestNotificationWorkerBlocksConfigurationAndFailsInvalid pins the two +// non-retry outcomes and proves an invalid payload never opens a Delivery +// gap (it is this build's own bug, not a Slack outage). +func TestNotificationWorkerBlocksConfigurationAndFailsInvalid(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{batches: [][]NotificationClaim{ + {nwClaim("intent-config", 1)}, + {nwClaim("intent-invalid", 1)}, + }} + deliverer := &nwDeliverer{deliver: func(intent model.NotificationIntent) (NotificationDelivery, error) { + if intent.ID == "intent-config" { + return NotificationDelivery{}, nwDeliveryError{class: DeliveryConfigurationBlocking, code: "invalid_auth"} + } + return NotificationDelivery{}, nwDeliveryError{class: DeliveryInvalid, code: "missing_text"} + }} + w := nwWorker(store, deliverer, now) + for i := 0; i < 2; i++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", i, err) + } + } + store.snapshot(func(s *nwStore) { + if len(s.blocked) != 1 || s.blocked[0] != "intent-config:invalid_auth" { + t.Fatalf("blocked = %v, want the configuration rejection", s.blocked) + } + if len(s.failed) != 1 || s.failed[0] != "intent-invalid:missing_text" { + t.Fatalf("failed = %v, want the invalid payload", s.failed) + } + if len(s.retried) != 0 { + t.Fatalf("retried = %v, want neither outcome to schedule a retry", s.retried) + } + if len(s.failures) != 1 || s.failures[0] != "invalid_auth" { + t.Fatalf("observed slack failures = %v, want only the configuration rejection", s.failures) + } + }) +} + +// TestNotificationWorkerUncertainSuccessConvergesToOneDeliveredIntent +// proves an uncertain external response followed by a successful retry +// reuses the identical client message id and converges locally to exactly +// one delivered intent — Slack itself stays at-least-once. +func TestNotificationWorkerUncertainSuccessConvergesToOneDeliveredIntent(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + claim := nwClaim("intent-uncertain", 1) + retryClaim := claim + retryClaim.Intent.AttemptCount = 2 + store := &nwStore{batches: [][]NotificationClaim{{claim}, {retryClaim}}} + + attempt := 0 + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + attempt++ + if attempt == 1 { + // Slack accepted the post but the response never arrived. + return NotificationDelivery{}, nwDeliveryError{class: DeliveryRetryable, code: "undecodable_response"} + } + return NotificationDelivery{Channel: "C", MessageTS: "100.1", DeliveredAs: "root"}, nil + }} + w := nwWorker(store, deliverer, now) + for i := 0; i < 2; i++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", i, err) + } + } + ids := deliverer.clientMessageIDs() + if len(ids) != 2 || ids[0] != ids[1] { + t.Fatalf("client message ids = %v, want the identical id reused on the retry", ids) + } + store.snapshot(func(s *nwStore) { + if len(s.delivered) != 1 || s.delivered[0] != "intent-uncertain" { + t.Fatalf("delivered = %v, want exactly one durable delivered intent", s.delivered) + } + if len(s.retried) != 1 { + t.Fatalf("retried = %v, want exactly the one uncertain attempt", s.retried) + } + }) +} + +// TestNotificationWorkerSupersededRootIsNotADeliveryFailure pins the Task 5 +// handoff (R4): losing a claim to a concurrent supersession is the expected +// outcome, never a retry, block, or failure. +func TestNotificationWorkerSupersededRootIsNotADeliveryFailure(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{ + batches: [][]NotificationClaim{{nwClaim("intent-superseded", 1)}}, + deliverAckErr: ErrNotificationIntentSuperseded, + } + w := nwWorker(store, &nwDeliverer{}, now) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + store.snapshot(func(s *nwStore) { + if len(s.delivered) != 0 || len(s.retried) != 0 || len(s.failed) != 0 || len(s.blocked) != 0 { + t.Fatalf("superseded ack wrote an outcome: delivered=%v retried=%v failed=%v blocked=%v", + s.delivered, s.retried, s.failed, s.blocked) + } + }) + if got := w.Stats().Superseded; got != 1 { + t.Fatalf("Stats().Superseded = %d, want 1", got) + } + if got := w.Stats().Failed; got != 0 { + t.Fatalf("Stats().Failed = %d, want 0", got) + } +} + +// TestNotificationWorkerRecordsStaleBroadcastAsDelayedThread proves the +// worker records the deliverer's revalidated delivery mode verbatim: a +// handoff found stale immediately before I/O lands as delayed_thread, never +// as a broadcast. +func TestNotificationWorkerRecordsStaleBroadcastAsDelayedThread(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + claim := nwClaim("intent-handoff", 1) + claim.Intent.EffectClass = model.EffectBroadcastHandoff + claim.Intent.SummaryVersion = nil + claim.Intent.RequiresRoot = true + store := &nwStore{batches: [][]NotificationClaim{{claim}}} + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{Channel: "C", MessageTS: "100.9", DeliveredAs: "delayed_thread"}, nil + }} + w := nwWorker(store, deliverer, now) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + store.snapshot(func(s *nwStore) { + if len(s.deliveredAs) != 1 || s.deliveredAs[0].DeliveredAs != "delayed_thread" { + t.Fatalf("recorded delivery = %+v, want delayed_thread", s.deliveredAs) + } + }) +} + +// ---------------------------------------------------------------------- +// Steps 5-7: probe, gap state machine, lifecycle. +// ---------------------------------------------------------------------- + +// TestNotificationWorkerProbesWhileAFailureWindowExists proves the worker +// probes on its first round and while a failure window, gap, or blocked +// configuration exists, and stops probing once Slack is healthy again. +func TestNotificationWorkerProbesWhileAFailureWindowExists(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{} + deliverer := &nwDeliverer{} + clock := &nwClock{at: now} + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", + Heartbeat: time.Hour, + Rand: func() float64 { return 0.5 }, + }, clock.now, nwLogger()) + + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("first RunOnce: %v", err) + } + if deliverer.probeCalls != 1 { + t.Fatalf("probe calls after the first round = %d, want exactly 1 startup probe", deliverer.probeCalls) + } + // Healthy: no more probing. + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("second RunOnce: %v", err) + } + if deliverer.probeCalls != 1 { + t.Fatalf("probe calls while healthy = %d, want no extra probe", deliverer.probeCalls) + } + // A failure window reopens probing, once the probe backoff has elapsed. + failedAt := now.Add(-time.Minute) + store.snapshot(func(s *nwStore) { s.state.FirstFailureAt = &failedAt }) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("third RunOnce: %v", err) + } + if deliverer.probeCalls != 1 { + t.Fatalf("probe calls before the probe backoff elapsed = %d, want no extra probe", deliverer.probeCalls) + } + clock.advance(time.Minute) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("fourth RunOnce: %v", err) + } + if deliverer.probeCalls != 2 { + t.Fatalf("probe calls with an open failure window = %d, want 2", deliverer.probeCalls) + } + store.snapshot(func(s *nwStore) { + if len(s.gapOpens) == 0 { + t.Fatal("a failure window must ask the store whether a gap is due") + } + if s.gapOpens[len(s.gapOpens)-1] != defaultDeliveryGapThreshold { + t.Fatalf("gap threshold = %s, want the fixed %s", s.gapOpens[len(s.gapOpens)-1], defaultDeliveryGapThreshold) + } + }) +} + +// TestNotificationWorkerRecoveryReactivatesConfigurationAndReplaysGap +// proves a successful probe closes the failure window, increments the +// durable configuration generation for blocked intents, and recovers the +// open gap — in that order, before any backlog claim. +func TestNotificationWorkerRecoveryReactivatesConfigurationAndReplaysGap(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + failedAt := now.Add(-10 * time.Minute) + generation := "gap-1" + store := &nwStore{ + state: SlackDeliveryState{ + FirstFailureAt: &failedAt, + OpenGapGeneration: &generation, + OpenGapStatus: "open", + ConfigurationGeneration: 3, + BlockedConfigurationCount: 2, + }, + recoverGap: generation, + recoverOK: true, + } + w := nwWorker(store, &nwDeliverer{}, now) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + store.snapshot(func(s *nwStore) { + if s.successes == 0 { + t.Fatal("a successful probe must close the failure window") + } + if len(s.reactivated) != 1 || s.reactivated[0] != 4 { + t.Fatalf("reactivated with generations %v, want exactly the incremented [4]", s.reactivated) + } + if s.recovers != 1 { + t.Fatalf("RecoverDeliveryGap calls = %d, want 1", s.recovers) + } + }) + stats := w.Stats() + if stats.GapsRecovered != 1 || stats.Reactivated != 1 { + t.Fatalf("stats = %+v, want one recovery and one reactivation", stats) + } +} + +// TestNotificationWorkerOpensAndCompletesGapGenerations proves the worker +// drives both ends of the durable generation lifecycle and counts them. +func TestNotificationWorkerOpensAndCompletesGapGenerations(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + failedAt := now.Add(-6 * time.Minute) + store := &nwStore{ + state: SlackDeliveryState{FirstFailureAt: &failedAt}, + openGap: true, + completeGap: "gap-1", + completeOK: true, + } + deliverer := &nwDeliverer{probeErr: errors.New("slack unreachable")} + w := nwWorker(store, deliverer, now) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + stats := w.Stats() + if stats.GapsOpened != 1 || stats.GapsCompleted != 1 || stats.ProbeFailures != 1 { + t.Fatalf("stats = %+v, want one gap opened, one completed, and one failed probe", stats) + } + store.snapshot(func(s *nwStore) { + if len(s.failures) != 1 { + t.Fatalf("observed slack failures = %v, want the failed probe to keep the window continuous", s.failures) + } + }) +} + +// TestNotificationWorkerHeartbeatLossAbandonsTheClaim proves a lost lease +// abandons the in-flight attempt rather than acknowledging an outcome onto +// a row that now belongs to someone else. +func TestNotificationWorkerHeartbeatLossAbandonsTheClaim(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{ + batches: [][]NotificationClaim{{nwClaim("intent-lease", 1)}}, + heartbeatErr: ErrNotificationClaimLost, + } + released := make(chan struct{}) + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + <-released // held until the heartbeat has had a chance to fail + return NotificationDelivery{Channel: "C", MessageTS: "1.1", DeliveredAs: "root"}, nil + }} + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", + Heartbeat: time.Millisecond, + Rand: func() float64 { return 0.5 }, + }, func() time.Time { return now }, nwLogger()) + + done := make(chan struct{}) + go func() { + defer close(done) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Errorf("RunOnce: %v", err) + } + }() + // Let the heartbeat fire and fail, then release the delivery. + for { + var beats int + store.snapshot(func(s *nwStore) { beats = s.heartbeats }) + if beats > 0 { + break + } + time.Sleep(time.Millisecond) + } + close(released) + <-done + + store.snapshot(func(s *nwStore) { + if len(s.delivered) != 0 || len(s.retried) != 0 || len(s.failed) != 0 { + t.Fatalf("a lost lease wrote an outcome: delivered=%v retried=%v failed=%v", s.delivered, s.retried, s.failed) + } + }) + if got := w.Stats().ClaimsLost; got != 1 { + t.Fatalf("Stats().ClaimsLost = %d, want 1", got) + } +} + +// TestNotificationWorkerStopRunsOneBoundedFinalPass proves R6's shutdown +// shape: the loop ends, exactly one more pass runs under the shutdown +// context, and no goroutine is left behind. +func TestNotificationWorkerStopRunsOneBoundedFinalPass(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{batches: [][]NotificationClaim{{nwClaim("intent-final", 1)}}} + w := NewNotificationWorker(store, &nwDeliverer{}, NotificationWorkerConfig{ + Owner: "notify-a", + Poll: time.Hour, // never ticks on its own during this test + Rand: func() float64 { return 0.5 }, + }, func() time.Time { return now }, nwLogger()) + + ctx := context.Background() + w.Start(ctx) + // The first round drains the queued batch; wait for it. + for { + var delivered int + store.snapshot(func(s *nwStore) { delivered = len(s.delivered) }) + if delivered == 1 { + break + } + time.Sleep(time.Millisecond) + } + + store.snapshot(func(s *nwStore) { s.batches = [][]NotificationClaim{{nwClaim("intent-shutdown", 1)}} }) + stopCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := w.Stop(stopCtx); err != nil { + t.Fatalf("Stop: %v", err) + } + store.snapshot(func(s *nwStore) { + if len(s.delivered) != 2 || s.delivered[1] != "intent-shutdown" { + t.Fatalf("delivered = %v, want the final pass to have drained intent-shutdown", s.delivered) + } + }) + // Stop is idempotent and never blocks a second time. + if err := w.Stop(stopCtx); err != nil { + t.Fatalf("second Stop: %v", err) + } +} + +// TestNotificationWorkerStopReleasesHeldClaims proves a claim still held +// when the final pass ends is handed straight back, so shutdown never +// leaves a durable obligation waiting out a full lease. +func TestNotificationWorkerStopReleasesHeldClaims(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{batches: [][]NotificationClaim{{nwClaim("intent-held", 1)}}} + w := NewNotificationWorker(store, &nwDeliverer{}, NotificationWorkerConfig{ + Owner: "notify-a", + Poll: time.Hour, + Rand: func() float64 { return 0.5 }, + }, func() time.Time { return now }, nwLogger()) + + // Simulate a claim the worker is still holding when Stop is called. + w.trackClaim(nwClaim("intent-held-open", 1)) + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := w.Stop(stopCtx); err != nil { + t.Fatalf("Stop: %v", err) + } + store.snapshot(func(s *nwStore) { + if len(s.released) != 1 || s.released[0] != "intent-held-open" { + t.Fatalf("released = %v, want the still-held claim handed back", s.released) + } + }) +} diff --git a/internal/store/notification_gaps.go b/internal/store/notification_gaps.go new file mode 100644 index 0000000..73cda93 --- /dev/null +++ b/internal/store/notification_gaps.go @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/alertint/alertint-agent/internal/situation" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 7: the durable, installation-level Delivery-gap lifecycle +// (migration 0018's slack_delivery_state singleton and slack_delivery_gaps +// generations). +// +// The whole machine is four transitions: +// +// healthy --first failure--> ordinary delay (first_failure_at set) +// ordinary delay --5 continuous minutes--> open generation +// open --successful readiness probe--> replaying (+ one recovery notice) +// replaying --backlog drained--> complete +// +// A success at any point clears the failure window, so only CONTINUOUS +// failure ever opens a gap. Generations are never deleted and never merged: +// a second outage during replay gets its own identity after its own full +// five-minute window. +// +// Which intents a generation is replaying is DERIVED, not stored: migration +// 0018 deliberately allows gap_generation only on the recovery notice +// itself, so "replayable" means "a pending Situation-scoped intent that +// already existed when this generation recovered" (created_at <= +// recovered_at). That keeps work committed after recovery — ordinary +// delivery, not replay — from holding a generation open forever. +// ---------------------------------------------------------------------- + +// GapSnapshot is one durable gap generation's bounded rendering facts: the +// outage interval and the backlog it delayed. It is exactly what +// cmd/alertint's Slack deliverer renders the ADR-0042 recovery notice from. +type GapSnapshot struct { + ID string + OpenedAt time.Time + RecoveredAt time.Time + AffectedSituationCount int + DelayedEffectCount int +} + +// GetDeliveryGap reads one gap generation's rendering facts. Returns +// ErrNotFound when no such generation exists. +func (s *Store) GetDeliveryGap(ctx context.Context, gapGeneration string) (GapSnapshot, error) { + if strings.TrimSpace(gapGeneration) == "" { + return GapSnapshot{}, errors.New("store: delivery gap read requires a generation id") + } + var openedAt string + var recoveredAt sql.NullString + snapshot := GapSnapshot{ID: gapGeneration} + err := s.db.QueryRowContext(ctx, ` + SELECT opened_at, recovered_at, affected_situation_count, delayed_effect_count + FROM slack_delivery_gaps WHERE id = ?`, gapGeneration). + Scan(&openedAt, &recoveredAt, &snapshot.AffectedSituationCount, &snapshot.DelayedEffectCount) + if errors.Is(err, sql.ErrNoRows) { + return GapSnapshot{}, ErrNotFound + } + if err != nil { + return GapSnapshot{}, fmt.Errorf("store: read delivery gap: %w", err) + } + opened, err := time.Parse(time.RFC3339Nano, openedAt) + if err != nil { + return GapSnapshot{}, fmt.Errorf("store: parse delivery gap opened_at: %w", err) + } + snapshot.OpenedAt = opened.UTC() + recovered, err := timePtr(recoveredAt) + if err != nil { + return GapSnapshot{}, fmt.Errorf("store: parse delivery gap recovered_at: %w", err) + } + if recovered != nil { + snapshot.RecoveredAt = *recovered + } + return snapshot, nil +} + +// GetSlackDeliveryState reads the installation-level Slack delivery health +// snapshot in one snapshot transaction, so the failure window, the current +// gap generation's status, and the blocked-configuration backlog can never +// disagree with each other. +func (s *Store) GetSlackDeliveryState(ctx context.Context) (situation.SlackDeliveryState, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return situation.SlackDeliveryState{}, fmt.Errorf("store: begin slack delivery state: %w", err) + } + defer func() { _ = tx.Rollback() }() + + state, err := slackDeliveryStateTx(ctx, tx) + if err != nil { + return situation.SlackDeliveryState{}, err + } + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents WHERE status = 'blocked_configuration'`). + Scan(&state.BlockedConfigurationCount); err != nil { + return situation.SlackDeliveryState{}, fmt.Errorf("store: count blocked notification intents: %w", err) + } + if err := tx.Commit(); err != nil { + return situation.SlackDeliveryState{}, fmt.Errorf("store: commit slack delivery state: %w", err) + } + return state, nil +} + +func slackDeliveryStateTx(ctx context.Context, tx *sql.Tx) (situation.SlackDeliveryState, error) { + var firstFailureAt, lastSuccessAt, openGap, lastWarningAt, gapStatus sql.NullString + var updatedAt string + var configurationGeneration int64 + err := tx.QueryRowContext(ctx, ` + SELECT st.first_failure_at, st.last_success_at, st.open_gap_generation, + st.configuration_generation, st.last_warning_at, st.updated_at, g.status + FROM slack_delivery_state st + LEFT JOIN slack_delivery_gaps g ON g.id = st.open_gap_generation + WHERE st.id = 1`). + Scan(&firstFailureAt, &lastSuccessAt, &openGap, &configurationGeneration, &lastWarningAt, &updatedAt, &gapStatus) + if err != nil { + return situation.SlackDeliveryState{}, fmt.Errorf("store: read slack delivery state: %w", err) + } + state := situation.SlackDeliveryState{ + ConfigurationGeneration: configurationGeneration, + OpenGapGeneration: stringPtr(openGap), + OpenGapStatus: gapStatus.String, + } + for _, f := range []struct { + src sql.NullString + dst **time.Time + }{ + {firstFailureAt, &state.FirstFailureAt}, {lastSuccessAt, &state.LastSuccessAt}, {lastWarningAt, &state.LastWarningAt}, + } { + parsed, err := timePtr(f.src) + if err != nil { + return situation.SlackDeliveryState{}, fmt.Errorf("store: parse slack delivery state instant: %w", err) + } + *f.dst = parsed + } + parsed, err := time.Parse(time.RFC3339Nano, updatedAt) + if err != nil { + return situation.SlackDeliveryState{}, fmt.Errorf("store: parse slack delivery state updated_at: %w", err) + } + state.UpdatedAt = parsed.UTC() + return state, nil +} + +// ObserveSlackFailure records one retryable or configuration failure against +// the current CONTINUOUS failure window. The first failure of a window +// anchors first_failure_at (and stamps the one bounded warning marker the +// console action trail emits at that moment); later failures leave the +// anchor exactly where it is, because the gap threshold measures continuous +// failure, not a sliding window. +func (s *Store) ObserveSlackFailure(ctx context.Context, errorClass string, now time.Time) error { + if err := validateNotificationErrorClass(errorClass); err != nil { + return err + } + stamp := canonicalTime(now.UTC()) + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state + SET first_failure_at = COALESCE(first_failure_at, ?), + last_warning_at = CASE WHEN first_failure_at IS NULL THEN ? ELSE last_warning_at END, + updated_at = ? + WHERE id = 1`, stamp, stamp, stamp); err != nil { + return fmt.Errorf("store: observe slack failure: %w", err) + } + return nil +} + +// ObserveSlackSuccess closes the current failure window. It does not touch +// an already-open generation: a gap is only ever retired by the recovery +// and completion transitions below, which have their own ordering +// obligations. +func (s *Store) ObserveSlackSuccess(ctx context.Context, now time.Time) error { + stamp := canonicalTime(now.UTC()) + if _, err := s.db.ExecContext(ctx, ` + UPDATE slack_delivery_state + SET first_failure_at = NULL, last_success_at = ?, updated_at = ? + WHERE id = 1`, stamp, stamp); err != nil { + return fmt.Errorf("store: observe slack success: %w", err) + } + return nil +} + +// OpenDueDeliveryGap opens one durable generation when — and only when — +// failures have been continuous for threshold. It reports whether it opened +// one. +// +// It is idempotent twice over: the generation's id is derived from the +// failure window's own anchor, and an already-open generation short-circuits +// the whole call. A generation still REPLAYING does not block a new one: a +// second outage mid-replay gets its own identity after its own full window, +// and the earlier generation still completes on its own drained-backlog +// condition. +func (s *Store) OpenDueDeliveryGap(ctx context.Context, now time.Time, threshold time.Duration) (bool, error) { + if threshold <= 0 { + return false, errors.New("store: delivery gap threshold must be positive") + } + now = now.UTC() + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("store: begin open delivery gap: %w", err) + } + defer func() { _ = tx.Rollback() }() + + state, err := slackDeliveryStateTx(ctx, tx) + if err != nil { + return false, err + } + if state.FirstFailureAt == nil || now.Sub(*state.FirstFailureAt) < threshold { + return false, nil + } + if state.OpenGapStatus == "open" { + return false, nil + } + + generation := situation.NewGapGenerationID(*state.FirstFailureAt) + var exists int + if err := tx.QueryRowContext(ctx, + `SELECT COUNT(*) FROM slack_delivery_gaps WHERE id = ?`, generation).Scan(&exists); err != nil { + return false, fmt.Errorf("store: check delivery gap generation: %w", err) + } + if exists > 0 { + return false, nil + } + + affected, delayed, err := replayableBacklogTx(ctx, tx, canonicalTime(now)) + if err != nil { + return false, err + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO slack_delivery_gaps (id, status, opened_at, affected_situation_count, delayed_effect_count) + VALUES (?, 'open', ?, ?, ?)`, + generation, canonicalTime(*state.FirstFailureAt), affected, delayed); err != nil { + return false, fmt.Errorf("store: insert delivery gap generation: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE slack_delivery_state SET open_gap_generation = ?, updated_at = ? WHERE id = 1`, + generation, canonicalTime(now)); err != nil { + return false, fmt.Errorf("store: point delivery state at the open generation: %w", err) + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("store: commit open delivery gap: %w", err) + } + return true, nil +} + +// RecoverDeliveryGap moves the oldest open generation into replay in one +// idempotent transaction: it closes the failure window, recomputes the +// backlog the notice reports, and creates the single claimable +// installation_gap_recovery intent that must deliver before any of that +// backlog does. It reports the generation it recovered. +// +// Closing the window here as well as in ObserveSlackSuccess is deliberate, +// not redundant: recovery is by definition proof that Slack answered, and +// the next generation's identity is derived from the NEXT window's anchor +// (NewGapGenerationID). A caller that recovered without first clearing the +// old anchor would leave a second outage unable to open a generation of its +// own, because it would keep computing the completed generation's id. +func (s *Store) RecoverDeliveryGap(ctx context.Context, now time.Time) (string, bool, error) { + now = now.UTC() + nowStr := canonicalTime(now) + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return "", false, fmt.Errorf("store: begin recover delivery gap: %w", err) + } + defer func() { _ = tx.Rollback() }() + + var generation string + err = tx.QueryRowContext(ctx, + `SELECT id FROM slack_delivery_gaps WHERE status = 'open' ORDER BY opened_at ASC, id ASC LIMIT 1`). + Scan(&generation) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("store: select open delivery gap: %w", err) + } + + affected, delayed, err := replayableBacklogTx(ctx, tx, nowStr) + if err != nil { + return "", false, err + } + notice, err := situation.GapRecoveryIntent(generation, now) + if err != nil { + return "", false, fmt.Errorf("store: build gap recovery notice: %w", err) + } + var existing int + if err := tx.QueryRowContext(ctx, + `SELECT COUNT(*) FROM notification_intents WHERE id = ?`, notice.ID).Scan(&existing); err != nil { + return "", false, fmt.Errorf("store: check gap recovery notice: %w", err) + } + if existing == 0 { + if err := insertNotificationIntentTx(ctx, tx, notice); err != nil { + return "", false, err + } + } + if _, err := tx.ExecContext(ctx, ` + UPDATE slack_delivery_gaps + SET status = 'replaying', recovered_at = ?, affected_situation_count = ?, delayed_effect_count = ?, + recovery_notice_intent_id = ? + WHERE id = ? AND status = 'open'`, + nowStr, affected, delayed, notice.ID, generation); err != nil { + return "", false, fmt.Errorf("store: move delivery gap into replay: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE slack_delivery_state SET first_failure_at = NULL, last_success_at = ?, updated_at = ? + WHERE id = 1`, nowStr, nowStr); err != nil { + return "", false, fmt.Errorf("store: close the recovered failure window: %w", err) + } + if err := tx.Commit(); err != nil { + return "", false, fmt.Errorf("store: commit recover delivery gap: %w", err) + } + return generation, true, nil +} + +// CompleteDeliveryGap completes the oldest replaying generation whose +// recovery notice has been delivered and whose replayable backlog has +// drained — never earlier. It reports the generation it completed. +// +// Only work that already existed when the generation recovered counts as +// replayable, so ordinary post-recovery delivery never holds a generation +// open, and a busy installation still reaches "complete". +func (s *Store) CompleteDeliveryGap(ctx context.Context, now time.Time) (string, bool, error) { + now = now.UTC() + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return "", false, fmt.Errorf("store: begin complete delivery gap: %w", err) + } + defer func() { _ = tx.Rollback() }() + + var generation, recoveredAt string + err = tx.QueryRowContext(ctx, ` + SELECT g.id, g.recovered_at + FROM slack_delivery_gaps g + JOIN notification_intents ni ON ni.id = g.recovery_notice_intent_id + WHERE g.status = 'replaying' AND ni.status = 'delivered' + ORDER BY g.opened_at ASC, g.id ASC + LIMIT 1`).Scan(&generation, &recoveredAt) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("store: select replaying delivery gap: %w", err) + } + + var remaining int + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents + WHERE status = 'pending' AND situation_id IS NOT NULL AND created_at <= ?`, recoveredAt). + Scan(&remaining); err != nil { + return "", false, fmt.Errorf("store: count replayable notification intents: %w", err) + } + if remaining > 0 { + return "", false, nil + } + + if _, err := tx.ExecContext(ctx, ` + UPDATE slack_delivery_gaps SET status = 'complete', completed_at = ? WHERE id = ? AND status = 'replaying'`, + canonicalTime(now), generation); err != nil { + return "", false, fmt.Errorf("store: complete delivery gap: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + UPDATE slack_delivery_state SET open_gap_generation = NULL, updated_at = ? + WHERE id = 1 AND open_gap_generation = ?`, canonicalTime(now), generation); err != nil { + return "", false, fmt.Errorf("store: clear the completed generation from delivery state: %w", err) + } + if err := tx.Commit(); err != nil { + return "", false, fmt.Errorf("store: commit complete delivery gap: %w", err) + } + return generation, true, nil +} + +// ReactivateConfigurationBlocked applies corrected Slack configuration: it +// advances the durable configuration generation to configurationGeneration +// and returns every blocked_configuration intent to pending, due now, with +// its attempt count preserved. It reports how many it reactivated. +// +// The generation is a compare-and-set, not a blind write: a value that does +// not advance the stored one reactivates nothing, so a restart loop can +// never replay the same correction twice. +func (s *Store) ReactivateConfigurationBlocked(ctx context.Context, configurationGeneration int64, + now time.Time) (int, error) { + if configurationGeneration <= 0 { + return 0, errors.New("store: configuration generation must be positive") + } + now = now.UTC() + nowStr := canonicalTime(now) + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("store: begin reactivate configuration-blocked intents: %w", err) + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.ExecContext(ctx, ` + UPDATE slack_delivery_state SET configuration_generation = ?, updated_at = ? + WHERE id = 1 AND configuration_generation < ?`, configurationGeneration, nowStr, configurationGeneration) + if err != nil { + return 0, fmt.Errorf("store: advance configuration generation: %w", err) + } + advanced, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count advanced configuration generation: %w", err) + } + if advanced == 0 { + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("store: commit unchanged configuration generation: %w", err) + } + return 0, nil + } + + reactivated, err := tx.ExecContext(ctx, ` + UPDATE notification_intents + SET status = 'pending', retry_at = ?, claim_owner = NULL, lease_expires_at = NULL + WHERE status = 'blocked_configuration'`, nowStr) + if err != nil { + return 0, fmt.Errorf("store: reactivate configuration-blocked intents: %w", err) + } + n, err := reactivated.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count reactivated notification intents: %w", err) + } + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("store: commit reactivate configuration-blocked intents: %w", err) + } + return int(n), nil +} + +// replayableBacklogTx counts the Situation-scoped delivery obligations +// outstanding as of asOf: how many distinct Situations, and how many +// effects. These are exactly the numbers the recovery notice reports. +func replayableBacklogTx(ctx context.Context, tx *sql.Tx, asOf string) (int, int, error) { + var affected, delayed int + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(DISTINCT situation_id), COUNT(*) + FROM notification_intents + WHERE status = 'pending' AND situation_id IS NOT NULL AND created_at <= ?`, asOf). + Scan(&affected, &delayed); err != nil { + return 0, 0, fmt.Errorf("store: count delayed notification backlog: %w", err) + } + return affected, delayed, nil +} diff --git a/internal/store/notification_gaps_test.go b/internal/store/notification_gaps_test.go new file mode 100644 index 0000000..1affd90 --- /dev/null +++ b/internal/store/notification_gaps_test.go @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 7, Steps 5-6: the durable Delivery-gap lifecycle. +// ---------------------------------------------------------------------- + +const snGapThreshold = 5 * time.Minute + +func snState(t *testing.T, st *Store) situation.SlackDeliveryState { + t.Helper() + state, err := st.GetSlackDeliveryState(context.Background()) + if err != nil { + t.Fatalf("GetSlackDeliveryState: %v", err) + } + return state +} + +func snFail(t *testing.T, st *Store, at time.Time) { + t.Helper() + if err := st.ObserveSlackFailure(context.Background(), "ratelimited", at); err != nil { + t.Fatalf("ObserveSlackFailure: %v", err) + } +} + +// TestDeliveryGapFirstFailureOpensOneContinuousWindow proves the first +// retryable failure stores first_failure_at, later failures leave that +// instant alone (the window is continuous, not sliding), and the durable +// warning marker is stamped exactly once. +func TestDeliveryGapFirstFailureOpensOneContinuousWindow(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + + if got := snState(t, st); got.FirstFailureAt != nil { + t.Fatalf("fresh delivery state first_failure_at = %v, want nil", got.FirstFailureAt) + } + snFail(t, st, now) + first := snState(t, st) + if first.FirstFailureAt == nil || !first.FirstFailureAt.Equal(now) { + t.Fatalf("first_failure_at = %v, want %s", first.FirstFailureAt, now) + } + if first.LastWarningAt == nil || !first.LastWarningAt.Equal(now) { + t.Fatalf("last_warning_at = %v, want the one bounded first-failure warning at %s", first.LastWarningAt, now) + } + snFail(t, st, now.Add(time.Minute)) + again := snState(t, st) + if !again.FirstFailureAt.Equal(now) { + t.Fatalf("first_failure_at moved to %v; the window must stay anchored at %s", again.FirstFailureAt, now) + } +} + +// TestDeliveryGapDoesNotOpenBeforeFiveContinuousMinutes pins the exact +// threshold: 4m59s of continuous failure is still an ordinary delay. +func TestDeliveryGapDoesNotOpenBeforeFiveContinuousMinutes(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + snFail(t, st, now) + + opened, err := st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold-time.Second), snGapThreshold) + if err != nil || opened { + t.Fatalf("OpenDueDeliveryGap at 4m59s = (%v, %v), want (false, nil)", opened, err) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM slack_delivery_gaps`); n != 0 { + t.Fatalf("gap generations before the threshold = %d, want 0", n) + } + opened, err = st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold), snGapThreshold) + if err != nil || !opened { + t.Fatalf("OpenDueDeliveryGap at exactly 5m = (%v, %v), want (true, nil)", opened, err) + } + // Idempotent: a second call while the same generation is open opens none. + opened, err = st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold+time.Minute), snGapThreshold) + if err != nil || opened { + t.Fatalf("second OpenDueDeliveryGap = (%v, %v), want (false, nil)", opened, err) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM slack_delivery_gaps`); n != 1 { + t.Fatalf("gap generations = %d, want exactly 1", n) + } +} + +// TestDeliveryGapSuccessResetsTheOrdinaryFailureWindow proves an +// intervening success makes the next failure start a fresh window, so +// intermittent failures never accumulate into a gap. +func TestDeliveryGapSuccessResetsTheOrdinaryFailureWindow(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + + snFail(t, st, now) + if err := st.ObserveSlackSuccess(ctx, now.Add(4*time.Minute)); err != nil { + t.Fatalf("ObserveSlackSuccess: %v", err) + } + cleared := snState(t, st) + if cleared.FirstFailureAt != nil { + t.Fatalf("first_failure_at after a success = %v, want nil", cleared.FirstFailureAt) + } + if cleared.LastSuccessAt == nil || !cleared.LastSuccessAt.Equal(now.Add(4*time.Minute)) { + t.Fatalf("last_success_at = %v, want the success instant", cleared.LastSuccessAt) + } + snFail(t, st, now.Add(4*time.Minute+time.Second)) + opened, err := st.OpenDueDeliveryGap(ctx, now.Add(8*time.Minute), snGapThreshold) + if err != nil || opened { + t.Fatalf("OpenDueDeliveryGap 4m into the second window = (%v, %v), want (false, nil)", opened, err) + } +} + +// TestDeliveryGapOpenRecordsAffectedBacklog proves the generation records +// the delayed backlog it is opening over, and that its rendering facts read +// back through GetDeliveryGap. +func TestDeliveryGapOpenRecordsAffectedBacklog(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + snSeedOneCycle(t, st, "group-gap-open-a", now) + snSeedOneCycle(t, st, "group-gap-open-b", now) + + snFail(t, st, now) + if opened, err := st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold), snGapThreshold); err != nil || !opened { + t.Fatalf("OpenDueDeliveryGap = (%v, %v), want (true, nil)", opened, err) + } + state := snState(t, st) + if state.OpenGapGeneration == nil || state.OpenGapStatus != "open" { + t.Fatalf("delivery state = %+v, want an open gap generation", state) + } + gap, err := st.GetDeliveryGap(ctx, *state.OpenGapGeneration) + if err != nil { + t.Fatalf("GetDeliveryGap: %v", err) + } + if !gap.OpenedAt.Equal(now) { + t.Fatalf("gap opened_at = %s, want the first failure instant %s", gap.OpenedAt, now) + } + if gap.AffectedSituationCount != 2 { + t.Fatalf("affected situation count = %d, want 2", gap.AffectedSituationCount) + } + if gap.DelayedEffectCount < 4 { + t.Fatalf("delayed effect count = %d, want every pending intent of both Situations", gap.DelayedEffectCount) + } + // While a gap is open nothing at all is claimable: Slack is down and + // the recovery notice must precede the backlog. + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(snGapThreshold), 5*time.Minute, 25) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + if len(claims) != 0 { + t.Fatalf("claimed %v inside an open gap, want nothing", snClasses(claims)) + } +} + +// TestDeliveryGapRecoveryCreatesOneNoticeAheadOfTheBacklog proves recovery +// moves the generation to replaying, creates exactly one +// installation_gap_recovery intent, and blocks every backlog claim until +// that notice is delivered. +func TestDeliveryGapRecoveryCreatesOneNoticeAheadOfTheBacklog(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + snSeedOneCycle(t, st, "group-gap-recover", now) + + snFail(t, st, now) + if _, err := st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold), snGapThreshold); err != nil { + t.Fatalf("OpenDueDeliveryGap: %v", err) + } + recoveredAt := now.Add(7 * time.Minute) + generation, ok, err := st.RecoverDeliveryGap(ctx, recoveredAt) + if err != nil || !ok || generation == "" { + t.Fatalf("RecoverDeliveryGap = (%q, %v, %v), want a recovered generation", generation, ok, err) + } + // Idempotent: nothing left to recover. + if _, again, err := st.RecoverDeliveryGap(ctx, recoveredAt.Add(time.Minute)); err != nil || again { + t.Fatalf("second RecoverDeliveryGap = (%v, %v), want (false, nil)", again, err) + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE effect_class = 'installation_gap_recovery'`); n != 1 { + t.Fatalf("recovery notices = %d, want exactly 1 per generation", n) + } + + notice := snClaimOne(t, st, recoveredAt.Add(time.Second)) + if notice.Intent.EffectClass != situationmodel.EffectInstallationGapRecovery { + t.Fatalf("claim while replaying = %q, want only the recovery notice", notice.Intent.EffectClass) + } + if notice.Intent.GapGeneration == nil || *notice.Intent.GapGeneration != generation { + t.Fatalf("recovery notice gap generation = %v, want %q", notice.Intent.GapGeneration, generation) + } + if err := st.MarkNotificationDelivered(ctx, notice, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: "70.7", DeliveredAs: "system"}, + recoveredAt.Add(time.Second)); err != nil { + t.Fatalf("MarkNotificationDelivered(notice): %v", err) + } + + backlog := snClaimOne(t, st, recoveredAt.Add(2*time.Second)) + if backlog.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("post-notice claim = %q, want the Situation's latest root projection", backlog.Intent.EffectClass) + } +} + +// TestDeliveryGapReplayCompletesOnlyWhenTheBacklogDrains proves the +// generation stays replaying until nothing replayable remains, and that +// work created after recovery never holds it open. +func TestDeliveryGapReplayCompletesOnlyWhenTheBacklogDrains(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-gap-replay", now) + + snFail(t, st, now) + if _, err := st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold), snGapThreshold); err != nil { + t.Fatalf("OpenDueDeliveryGap: %v", err) + } + recoveredAt := now.Add(7 * time.Minute) + generation, _, err := st.RecoverDeliveryGap(ctx, recoveredAt) + if err != nil { + t.Fatalf("RecoverDeliveryGap: %v", err) + } + + at := recoveredAt + for round := 0; round < 6; round++ { + at = at.Add(time.Second) + claims, err := st.ClaimNotificationIntents(ctx, snOwner, at, 5*time.Minute, 25) + if err != nil { + t.Fatalf("claim round %d: %v", round, err) + } + if len(claims) == 0 { + break + } + // Work is still outstanding, so the generation may not complete. + if _, done, err := st.CompleteDeliveryGap(ctx, at); err != nil { + t.Fatalf("CompleteDeliveryGap: %v", err) + } else if done { + t.Fatalf("generation completed while %d replayable intents were still in flight", len(claims)) + } + for i, c := range claims { + snDeliver(t, st, c, "10"+string(rune('0'+round))+"."+string(rune('0'+i)), at) + } + } + + completed, done, err := st.CompleteDeliveryGap(ctx, at.Add(time.Minute)) + if err != nil || !done || completed != generation { + t.Fatalf("CompleteDeliveryGap with a drained backlog = (%q, %v, %v), want (%q, true, nil)", + completed, done, err, generation) + } + var status string + if err := st.db.QueryRowContext(ctx, `SELECT status FROM slack_delivery_gaps WHERE id = ?`, generation). + Scan(&status); err != nil { + t.Fatalf("read gap status: %v", err) + } + if status != "complete" { + t.Fatalf("gap status = %q, want complete", status) + } + if state := snState(t, st); state.OpenGapGeneration != nil { + t.Fatalf("delivery state still names gap %v after completion", state.OpenGapGeneration) + } + // New work committed after recovery is ordinary delivery, not replay. + _ = snCommit(t, st, sitID, &first, at.Add(2*time.Minute)) + if _, done, err := st.CompleteDeliveryGap(ctx, at.Add(3*time.Minute)); err != nil || done { + t.Fatalf("CompleteDeliveryGap after completion = (%v, %v), want (false, nil)", done, err) + } +} + +// TestDeliveryGapReplaySecondOutageStartsADistinctGeneration proves a +// failure during replay does not reuse or reopen the first generation: it +// needs its own continuous five-minute window and gets its own identity. +func TestDeliveryGapReplaySecondOutageStartsADistinctGeneration(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + snSeedOneCycle(t, st, "group-gap-second", now) + + snFail(t, st, now) + if _, err := st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold), snGapThreshold); err != nil { + t.Fatalf("OpenDueDeliveryGap: %v", err) + } + firstGeneration, _, err := st.RecoverDeliveryGap(ctx, now.Add(7*time.Minute)) + if err != nil { + t.Fatalf("RecoverDeliveryGap: %v", err) + } + + // Slack fails again mid-replay. + secondWindow := now.Add(8 * time.Minute) + snFail(t, st, secondWindow) + if opened, err := st.OpenDueDeliveryGap(ctx, secondWindow.Add(snGapThreshold-time.Second), snGapThreshold); err != nil || opened { + t.Fatalf("second gap before its own five minutes = (%v, %v), want (false, nil)", opened, err) + } + opened, err := st.OpenDueDeliveryGap(ctx, secondWindow.Add(snGapThreshold), snGapThreshold) + if err != nil || !opened { + t.Fatalf("second gap at its own five minutes = (%v, %v), want (true, nil)", opened, err) + } + state := snState(t, st) + if state.OpenGapGeneration == nil || *state.OpenGapGeneration == firstGeneration { + t.Fatalf("open gap generation = %v, want a distinct generation from %q", state.OpenGapGeneration, firstGeneration) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM slack_delivery_gaps`); n != 2 { + t.Fatalf("gap generations = %d, want 2 distinct ones", n) + } +} + +// TestDeliveryGapConfigurationGenerationReactivatesBlockedIntents proves +// corrected startup configuration increments the durable configuration +// generation and returns blocked intents to pending exactly once. +func TestDeliveryGapConfigurationGenerationReactivatesBlockedIntents(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + snSeedOneCycle(t, st, "group-gap-config", now) + + claim := snClaimOne(t, st, now) + if err := st.BlockNotificationConfiguration(ctx, claim, "invalid_auth", now); err != nil { + t.Fatalf("BlockNotificationConfiguration: %v", err) + } + state := snState(t, st) + if state.BlockedConfigurationCount != 1 { + t.Fatalf("blocked configuration count = %d, want 1", state.BlockedConfigurationCount) + } + if state.ConfigurationGeneration != 0 { + t.Fatalf("initial configuration generation = %d, want 0", state.ConfigurationGeneration) + } + + n, err := st.ReactivateConfigurationBlocked(ctx, state.ConfigurationGeneration+1, now.Add(time.Minute)) + if err != nil || n != 1 { + t.Fatalf("ReactivateConfigurationBlocked = (%d, %v), want (1, nil)", n, err) + } + bumped := snState(t, st) + if bumped.ConfigurationGeneration != 1 || bumped.BlockedConfigurationCount != 0 { + t.Fatalf("delivery state after reactivation = %+v, want generation 1 and no blocked intents", bumped) + } + // Replaying the same generation is a no-op, so a restart loop can never + // reactivate the same correction twice. + if n, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(2*time.Minute)); err != nil || n != 0 { + t.Fatalf("replayed ReactivateConfigurationBlocked = (%d, %v), want (0, nil)", n, err) + } + reactivated := snIntent(t, st, claim.Intent.ID) + if reactivated.Status != situationmodel.IntentPending || reactivated.AttemptCount != 1 { + t.Fatalf("reactivated intent = %+v, want pending with its attempts preserved", reactivated) + } +} diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go new file mode 100644 index 0000000..516f772 --- /dev/null +++ b/internal/store/situation_notifications.go @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 7: the fenced notification-intent claim/acknowledge surface +// migration 0018's ledger exists for. Every write here is fenced on the +// full (id, status='pending', claim_owner, claim_token) tuple at once, so a +// stale acknowledgement — an expired lease that was swept, a claim another +// worker reclaimed, or a root projection a concurrent authoritative commit +// superseded (R4) — changes exactly zero rows and says so, rather than +// silently writing a delivery outcome onto a row that has moved on. +// +// No Slack call ever happens inside these transactions: the worker +// (internal/situation/notification_worker.go) claims, calls out, and comes +// back to acknowledge. +// ---------------------------------------------------------------------- + +// These are situation's own sentinels rather than store-local mirrors: the +// claim contract (situation.NotificationClaim) already crosses this +// boundary in both directions, so a second vocabulary plus a translation +// adapter would only create somewhere for the two to drift apart. Mirrors +// store.ErrNotFound = situationmodel.ErrNotFound. +var ( + // ErrNotificationClaimLost is situation.ErrNotificationClaimLost. + ErrNotificationClaimLost = situation.ErrNotificationClaimLost + // ErrNotificationIntentSuperseded is + // situation.ErrNotificationIntentSuperseded. + ErrNotificationIntentSuperseded = situation.ErrNotificationIntentSuperseded +) + +// notificationClaimOrder is the exact claim ordering the plan names: gap +// generation first (the installation recovery notice precedes every +// Situation's backlog), then Situation, then — within a Situation — the +// coalescible root projection ahead of the immutable journal, then +// Transition sequence, then creation identity. +// +// notificationClassRank is that within-Situation rank as a SQL expression: +// root_sync (0) before thread_append (1) before broadcast_handoff (2), so a +// root edit for a handoff always delivers before its broadcast reply +// (spec.md "Local idempotency, external delivery, and ordering", rule 3). +const ( + notificationClassRank = `CASE ni.effect_class WHEN 'root_sync' THEN 0 WHEN 'thread_append' THEN 1 ELSE 2 END` + notificationClaimOrder = `ORDER BY (gap_generation IS NULL) ASC, gap_generation ASC, situation_id ASC, class_rank ASC, transition_sequence ASC, id ASC` +) + +// validateNotificationClaim rejects a claim that cannot fence anything. +func validateNotificationClaim(claim situation.NotificationClaim) error { + if strings.TrimSpace(claim.Intent.ID) == "" { + return errors.New("store: notification acknowledgement requires an intent id") + } + if strings.TrimSpace(claim.ClaimOwner) == "" { + return errors.New("store: notification acknowledgement requires a claim owner") + } + if claim.ClaimToken <= 0 { + return errors.New("store: notification acknowledgement requires a positive claim token") + } + return nil +} + +// validateNotificationErrorClass enforces last_error_class's closed +// lowercase-identifier shape, exactly as the alert dispatch ledger does: it +// is a classification column, never anywhere a raw error message (which +// could embed a URL, a header value, or a provider body) can land. +func validateNotificationErrorClass(class string) error { + if class == "" { + return errors.New("store: notification error class is required") + } + if len(class) > maxErrorClassLength { + return fmt.Errorf("store: notification error class exceeds %d characters", maxErrorClassLength) + } + if !errorClassPattern.MatchString(class) { + return errors.New("store: notification error class must be a lowercase identifier (e.g. \"ratelimited\"), not raw error text") + } + return nil +} + +// ClaimNotificationIntents leases the currently deliverable notification +// intents in one immediate transaction, newest lease wins. +// +// Deliverable means all of: +// +// - pending, and either unclaimed or holding an expired lease; +// - due (no retry time, or one that has passed); +// - root-ready: a thread_append/broadcast_handoff is claimable only once +// its Situation's root coordinates are durably published, so a reply +// never consumes a delivery attempt waiting for a root, and a failed or +// configuration-blocked root leaves its dependents waiting rather than +// dead-lettering them; and +// - the HEAD of its Situation's queue. Exactly one intent per Situation +// is claimable at a time, ranked root projection first and then by +// Transition sequence, so a later immutable entry can never pass an +// earlier pending one — including one merely waiting out a retry delay. +// +// A gap generation gates the whole claim: while one is open nothing is +// claimable at all (Slack is down and the recovery notice must precede the +// backlog), and while one is replaying only its own undelivered recovery +// notice is. A recovery notice that ends up blocked or failed therefore +// holds the backlog — deliberately, since the notice is the operator's only +// signal that the history arriving next is delayed; it is reactivated by a +// corrected configuration generation or an explicit redrive, exactly like +// any other intent. +// +// Claiming increments claim_token (fencing every prior holder out) and +// attempt_count. It calls nothing outbound. +func (s *Store) ClaimNotificationIntents(ctx context.Context, owner string, now time.Time, + lease time.Duration, limit int) ([]situation.NotificationClaim, error) { + if strings.TrimSpace(owner) == "" || lease <= 0 || limit <= 0 { + return nil, errors.New("store: notification claim requires owner, positive lease, and positive limit") + } + now = now.UTC() + nowStr := canonicalTime(now) + leaseExpires := canonicalTime(now.Add(lease)) + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("store: begin claim notification intents: %w", err) + } + defer func() { _ = tx.Rollback() }() + + gate, err := deliveryGapGateTx(ctx, tx) + if err != nil { + return nil, err + } + if gate.blocked { + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("store: commit gated notification claim: %w", err) + } + return []situation.NotificationClaim{}, nil + } + + ids, err := dueNotificationIntentIDsTx(ctx, tx, gate, nowStr, limit) + if err != nil { + return nil, err + } + if len(ids) == 0 { + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("store: commit empty notification claim: %w", err) + } + return []situation.NotificationClaim{}, nil + } + + placeholders, args := inPlaceholders(ids) + updateArgs := append([]any{owner, leaseExpires}, args...) + if _, err := tx.ExecContext(ctx, ` + UPDATE notification_intents + SET claim_owner = ?, lease_expires_at = ?, claim_token = claim_token + 1, attempt_count = attempt_count + 1 + WHERE id IN (`+placeholders+`)`, updateArgs...); err != nil { // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + return nil, fmt.Errorf("store: claim notification intents: %w", err) + } + + claims, err := loadClaimedNotificationIntentsTx(ctx, tx, ids, owner) + if err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("store: commit claim notification intents: %w", err) + } + return claims, nil +} + +// deliveryGapGate is one claim round's gap decision: blocked means nothing +// is claimable; noticeIDs, when non-empty, restricts the round to exactly +// those undelivered recovery notices. +type deliveryGapGate struct { + blocked bool + noticeIDs []string +} + +// deliveryGapGateTx reads the gap lifecycle's effect on claimability. +func deliveryGapGateTx(ctx context.Context, tx *sql.Tx) (deliveryGapGate, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT g.status, COALESCE(g.recovery_notice_intent_id, ''), COALESCE(ni.status, '') + FROM slack_delivery_gaps g + LEFT JOIN notification_intents ni ON ni.id = g.recovery_notice_intent_id + WHERE g.status IN ('open','replaying') + ORDER BY g.opened_at ASC`) + if err != nil { + return deliveryGapGate{}, fmt.Errorf("store: read delivery gap gate: %w", err) + } + defer func() { _ = rows.Close() }() + + gate := deliveryGapGate{} + for rows.Next() { + var status, noticeID, noticeStatus string + if err := rows.Scan(&status, ¬iceID, ¬iceStatus); err != nil { + return deliveryGapGate{}, fmt.Errorf("store: scan delivery gap gate: %w", err) + } + if status == "open" { + // Slack is down and the ADR-0042 notice has to precede the + // backlog, so no effect is claimable at all until a readiness + // probe recovers this generation. + return deliveryGapGate{blocked: true}, nil + } + if noticeID != "" && noticeStatus != string(situationmodel.IntentDelivered) { + gate.noticeIDs = append(gate.noticeIDs, noticeID) + } + } + if err := rows.Err(); err != nil { + return deliveryGapGate{}, fmt.Errorf("store: iterate delivery gap gate: %w", err) + } + return gate, nil +} + +// dueNotificationIntentIDsTx selects the ids this round may claim, in +// notificationClaimOrder. +func dueNotificationIntentIDsTx(ctx context.Context, tx *sql.Tx, gate deliveryGapGate, + nowStr string, limit int) ([]string, error) { + if len(gate.noticeIDs) > 0 { + placeholders, args := inPlaceholders(gate.noticeIDs) + args = append(args, nowStr, nowStr, limit) + rows, err := tx.QueryContext(ctx, ` + SELECT id FROM notification_intents + WHERE id IN (`+placeholders+`) AND status = 'pending' + AND (claim_owner IS NULL OR lease_expires_at <= ?) + AND (retry_at IS NULL OR retry_at <= ?) + ORDER BY created_at ASC, id ASC + LIMIT ?`, args...) // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(noticeIDs); every value is bound + if err != nil { + return nil, fmt.Errorf("store: select due recovery notice: %w", err) + } + ids, err := scanStringRows(rows) + if err != nil { + return nil, fmt.Errorf("store: read due recovery notice ids: %w", err) + } + return ids, nil + } + + rows, err := tx.QueryContext(ctx, ` + WITH ranked AS ( + SELECT ni.id AS id, + ni.gap_generation AS gap_generation, + ni.situation_id AS situation_id, + ni.transition_sequence AS transition_sequence, + `+notificationClassRank+` AS class_rank, + (ni.claim_owner IS NULL OR ni.lease_expires_at <= ?) AS unleased, + (ni.retry_at IS NULL OR ni.retry_at <= ?) AS due, + (ni.requires_root = 0 OR (s.slack_channel IS NOT NULL AND s.slack_root_ts IS NOT NULL)) AS root_ready, + ROW_NUMBER() OVER ( + PARTITION BY ni.situation_id + ORDER BY `+notificationClassRank+` ASC, ni.transition_sequence ASC, ni.id ASC + ) AS rn + FROM notification_intents ni + LEFT JOIN situations s ON s.id = ni.situation_id + WHERE ni.status = 'pending' + ) + SELECT id FROM ranked + WHERE (situation_id IS NULL OR rn = 1) AND unleased AND due AND root_ready + `+notificationClaimOrder+` + LIMIT ?`, nowStr, nowStr, limit) + if err != nil { + return nil, fmt.Errorf("store: select due notification intents: %w", err) + } + ids, err := scanStringRows(rows) + if err != nil { + return nil, fmt.Errorf("store: read due notification intent ids: %w", err) + } + return ids, nil +} + +// loadClaimedNotificationIntentsTx re-reads the just-claimed rows in the +// same deterministic order they were selected in. A bare UPDATE ... +// RETURNING does not guarantee it preserves the subquery's ORDER BY, and +// every column this orders by is one claiming never touches. +func loadClaimedNotificationIntentsTx(ctx context.Context, tx *sql.Tx, ids []string, + owner string) ([]situation.NotificationClaim, error) { + placeholders, args := inPlaceholders(ids) + args = append(args, owner) + query := ` + SELECT ` + qualifyColumns(notificationIntentColumns, "ni") + `, + ` + notificationClassRank + ` AS class_rank + FROM notification_intents ni + WHERE ni.id IN (` + placeholders + `) AND ni.claim_owner = ? + ` + notificationClaimOrder // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("store: read claimed notification intents: %w", err) + } + defer func() { _ = rows.Close() }() + + out := make([]situation.NotificationClaim, 0, len(ids)) + for rows.Next() { + var classRank int + intent, err := scanNotificationIntent(suffixedScanner{rows: rows, suffix: []any{&classRank}}) + if err != nil { + return nil, fmt.Errorf("store: scan claimed notification intent: %w", err) + } + out = append(out, situation.NotificationClaim{Intent: intent, ClaimOwner: owner, ClaimToken: intent.ClaimToken}) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate claimed notification intents: %w", err) + } + return out, nil +} + +// suffixedScanner lets scanNotificationIntent consume a row carrying extra +// TRAILING columns (the ORDER BY's own class_rank), mirroring +// prefixedScanner one direction over. +type suffixedScanner struct { + rows *sql.Rows + suffix []any +} + +func (p suffixedScanner) Scan(dest ...any) error { + all := make([]any, 0, len(dest)+len(p.suffix)) + all = append(all, dest...) + all = append(all, p.suffix...) + return p.rows.Scan(all...) +} + +// inPlaceholders builds a "?,?,..." run of len(values) and the matching +// bound argument slice. +func inPlaceholders(values []string) (string, []any) { + parts := make([]string, len(values)) + args := make([]any, 0, len(values)) + for i, v := range values { + parts[i] = "?" + args = append(args, v) + } + return strings.Join(parts, ","), args +} + +// ---------------------------------------------------------------------- +// Fenced acknowledgements. +// ---------------------------------------------------------------------- + +// fencedNotificationAck applies one fenced lifecycle write and resolves a +// zero-row result into the right typed reason. +func (s *Store) fencedNotificationAck(ctx context.Context, claim situation.NotificationClaim, + setClause string, args ...any) error { + if err := validateNotificationClaim(claim); err != nil { + return err + } + args = append(args, claim.Intent.ID, claim.ClaimOwner, claim.ClaimToken) + res, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents SET `+setClause+` + WHERE id = ? AND status = 'pending' AND claim_owner = ? AND claim_token = ?`, args...) // #nosec G202 -- setClause is a package-local constant expression; every value is bound + if err != nil { + return fmt.Errorf("store: acknowledge notification intent: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("store: count acknowledged notification intent: %w", err) + } + if n != 1 { + return classifyLostNotificationClaim(ctx, s.db, claim.Intent.ID) + } + return nil +} + +// rowQueryer is the subset of *sql.DB / *sql.Tx classifyLostNotificationClaim +// needs. Taking it explicitly matters: the store runs on a single pooled +// connection (SetMaxOpenConns(1)), so a caller that already holds an open +// transaction MUST classify through that transaction rather than through +// s.db, which would wait forever for a connection it is itself holding. +type rowQueryer interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// classifyLostNotificationClaim explains a zero-row fenced write: a +// superseded root projection (the expected R4 race, and a status a +// delivered write could never reach anyway — migration 0018's +// supersession CHECKs forbid it) or an ordinary lost claim. +func classifyLostNotificationClaim(ctx context.Context, q rowQueryer, intentID string) error { + var status string + err := q.QueryRowContext(ctx, `SELECT status FROM notification_intents WHERE id = ?`, intentID).Scan(&status) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("store: notification intent %s: %w", intentID, ErrNotFound) + } + if err != nil { + return fmt.Errorf("store: classify lost notification claim: %w", err) + } + if situationmodel.IntentStatus(status) == situationmodel.IntentSuperseded { + return ErrNotificationIntentSuperseded + } + return ErrNotificationClaimLost +} + +// MarkNotificationDelivered records one fenced successful delivery and, for +// a root projection only, the Situation's durable root coordinates — the +// single place slack_channel/slack_root_ts is ever written, and only when +// the matching fenced root delivery is acknowledged. +func (s *Store) MarkNotificationDelivered(ctx context.Context, claim situation.NotificationClaim, + delivery situation.NotificationDelivery, now time.Time) error { + if err := validateNotificationClaim(claim); err != nil { + return err + } + if strings.TrimSpace(delivery.Channel) == "" || strings.TrimSpace(delivery.MessageTS) == "" { + return errors.New("store: notification delivery requires channel and message timestamp") + } + switch delivery.DeliveredAs { + case "root", "thread", "broadcast", "delayed_thread", "system": + default: + return fmt.Errorf("store: notification delivery mode %q is not one of root|thread|broadcast|delayed_thread|system", + delivery.DeliveredAs) + } + now = now.UTC() + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("store: begin mark notification delivered: %w", err) + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.ExecContext(ctx, ` + UPDATE notification_intents + SET status = 'delivered', delivered_at = ?, channel = ?, message_ts = ?, delivered_as = ?, + claim_owner = NULL, lease_expires_at = NULL, retry_at = NULL + WHERE id = ? AND status = 'pending' AND claim_owner = ? AND claim_token = ?`, + canonicalTime(now), delivery.Channel, delivery.MessageTS, delivery.DeliveredAs, + claim.Intent.ID, claim.ClaimOwner, claim.ClaimToken) + if err != nil { + return fmt.Errorf("store: mark notification delivered: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("store: count delivered notification intent: %w", err) + } + if n != 1 { + return classifyLostNotificationClaim(ctx, tx, claim.Intent.ID) + } + + // Which Situation's root this is comes from the intent ROW, never from + // the caller's copy of it: the fence above already proved this row is + // ours, so the row is also the authority on what it points at. + if claim.Intent.EffectClass == situationmodel.EffectRootSync { + if _, err := tx.ExecContext(ctx, ` + UPDATE situations SET slack_channel = ?, slack_root_ts = ? + WHERE id = (SELECT situation_id FROM notification_intents WHERE id = ? AND effect_class = 'root_sync')`, + delivery.Channel, delivery.MessageTS, claim.Intent.ID); err != nil { + return fmt.Errorf("store: persist situation root coordinates: %w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("store: commit mark notification delivered: %w", err) + } + return nil +} + +// RetryNotificationIntent releases a claimed intent back for a later retry. +// It keeps the intent pending and keeps its attempt count: there is no +// attempt ceiling anywhere in this lifecycle. +func (s *Store) RetryNotificationIntent(ctx context.Context, claim situation.NotificationClaim, + class string, retryAt time.Time) error { + if err := validateNotificationErrorClass(class); err != nil { + return err + } + if retryAt.IsZero() { + return errors.New("store: notification retry time is required") + } + return s.fencedNotificationAck(ctx, claim, + `claim_owner = NULL, lease_expires_at = NULL, last_error_class = ?, retry_at = ?`, + class, canonicalTime(retryAt)) +} + +// BlockNotificationConfiguration records a definite Slack configuration +// rejection. The intent stays durable with its attempts intact and no retry +// time: it waits for a corrected configuration generation, never for an +// exhausted attempt budget. +func (s *Store) BlockNotificationConfiguration(ctx context.Context, claim situation.NotificationClaim, + class string, _ time.Time) error { + if err := validateNotificationErrorClass(class); err != nil { + return err + } + return s.fencedNotificationAck(ctx, claim, + `status = 'blocked_configuration', claim_owner = NULL, lease_expires_at = NULL, + last_error_class = ?, retry_at = NULL`, class) +} + +// FailNotificationIntent records the one non-recoverable outcome: an +// invalid durable intent or another programming/data error. It is +// operator-visible and explicitly redriveable, and it never cascades — a +// failed root leaves its dependents pending, not dead-lettered. +func (s *Store) FailNotificationIntent(ctx context.Context, claim situation.NotificationClaim, + class string, _ time.Time) error { + if err := validateNotificationErrorClass(class); err != nil { + return err + } + return s.fencedNotificationAck(ctx, claim, + `status = 'failed', claim_owner = NULL, lease_expires_at = NULL, last_error_class = ?, retry_at = NULL`, + class) +} + +// HeartbeatNotificationClaim moves a live claim's lease deadline forward +// without changing anything else about the intent. +func (s *Store) HeartbeatNotificationClaim(ctx context.Context, claim situation.NotificationClaim, + now time.Time, lease time.Duration) error { + if lease <= 0 { + return errors.New("store: notification heartbeat requires a positive lease") + } + return s.fencedNotificationAck(ctx, claim, `lease_expires_at = ?`, canonicalTime(now.UTC().Add(lease))) +} + +// ReleaseNotificationClaim hands a claim straight back, still pending and +// still due — the clean shutdown path, so a stopped worker never leaves a +// durable obligation waiting out a full lease. +func (s *Store) ReleaseNotificationClaim(ctx context.Context, claim situation.NotificationClaim, _ time.Time) error { + return s.fencedNotificationAck(ctx, claim, `claim_owner = NULL, lease_expires_at = NULL`) +} + +// RecoverExpiredNotificationClaims sweeps every claim whose lease has +// expired back to unclaimed, so a crashed worker's in-flight intents become +// claimable again. It never changes status or attempts. +func (s *Store) RecoverExpiredNotificationClaims(ctx context.Context, now time.Time) (int, error) { + res, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents + SET claim_owner = NULL, lease_expires_at = NULL + WHERE status = 'pending' AND claim_owner IS NOT NULL AND lease_expires_at IS NOT NULL + AND lease_expires_at <= ?`, canonicalTime(now.UTC())) + if err != nil { + return 0, fmt.Errorf("store: recover expired notification claims: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count recovered notification claims: %w", err) + } + return int(n), nil +} + +// RedriveFailedNotificationIntent returns one explicitly-redriven failed +// intent to pending, due now, with its attempt count preserved. It is the +// only way out of `failed`, and the way a failed root releases the +// dependent history waiting behind it. +func (s *Store) RedriveFailedNotificationIntent(ctx context.Context, intentID string, now time.Time) error { + if strings.TrimSpace(intentID) == "" { + return errors.New("store: notification redrive requires an intent id") + } + res, err := s.db.ExecContext(ctx, ` + UPDATE notification_intents + SET status = 'pending', retry_at = ?, claim_owner = NULL, lease_expires_at = NULL + WHERE id = ? AND status = 'failed'`, canonicalTime(now.UTC()), intentID) + if err != nil { + return fmt.Errorf("store: redrive failed notification intent: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("store: count redriven notification intent: %w", err) + } + if n != 1 { + return fmt.Errorf("store: notification intent %s is not in failed status: %w", intentID, ErrNotFound) + } + return nil +} + +// GetSituationRootCoordinates reads situationID's durable Slack root +// coordinates. ok is false when no root has been delivered yet — the +// coordinates are nullable COLUMNS on an existing row, so their absence is +// a normal state, not a missing record. +func (s *Store) GetSituationRootCoordinates(ctx context.Context, situationID string) (string, string, bool, error) { + if strings.TrimSpace(situationID) == "" { + return "", "", false, errors.New("store: situation root coordinates require a situation id") + } + var channel, ts sql.NullString + err := s.db.QueryRowContext(ctx, + `SELECT slack_channel, slack_root_ts FROM situations WHERE id = ?`, situationID).Scan(&channel, &ts) + if errors.Is(err, sql.ErrNoRows) { + return "", "", false, ErrNotFound + } + if err != nil { + return "", "", false, fmt.Errorf("store: read situation root coordinates: %w", err) + } + if !channel.Valid || !ts.Valid { + return "", "", false, nil + } + return channel.String, ts.String, true, nil +} diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go new file mode 100644 index 0000000..11144ec --- /dev/null +++ b/internal/store/situation_notifications_test.go @@ -0,0 +1,638 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 7 fixtures. Every helper is prefixed `sn` (situation +// notifications) so it never collides with Task 5's `sh` history fixtures +// living in the same package. +// ---------------------------------------------------------------------- + +const snOwner = "notify-a" + +// snCommit runs one real fenced controller commit against sitID and returns +// it. A nil prior means "the Situation's first cycle" (its root has never +// been published); a non-nil prior continues that history with a materially +// changed Operator contract, which is what makes the second cycle produce a +// new root projection plus its own immutable journal entry. +func snCommit(t *testing.T, st *Store, sitID string, prior *situation.ControllerCommit, now time.Time) situation.ControllerCommit { + t.Helper() + contract := shRunningTriageContract(now.Add(time.Minute)) + if prior != nil { + contract = shOperatorContract(now.Add(time.Minute)) + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + } + claim := claimSituation(t, st, sitID, "controller-a", now) + cycle := shPrepare(t, claim, contract, situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + if prior != nil { + last := prior.History.Transitions[len(prior.History.Transitions)-1] + cycle.Change.PriorTransition = &last + cycle.Change.PriorSummary = prior.History.Summary + cycle.Publish.PriorTransition = &last + cycle.Publish.RootPublished = true + } + commit := shDerive(t, cycle) + if err := st.CommitController(context.Background(), claim, commit); err != nil { + t.Fatalf("CommitController: %v", err) + } + return commit +} + +// snSeedOneCycle creates a Situation with exactly one committed cycle: a +// pending root_sync plus one immutable thread_append at sequence 1. +func snSeedOneCycle(t *testing.T, st *Store, group string, now time.Time) (string, situation.ControllerCommit) { + t.Helper() + sitID := newSituationForGroup(t, st, group, now) + return sitID, snCommit(t, st, sitID, nil, now) +} + +func snIntent(t *testing.T, st *Store, id string) situationmodel.NotificationIntent { + t.Helper() + intent, err := st.GetNotificationIntent(context.Background(), id) + if err != nil { + t.Fatalf("GetNotificationIntent(%s): %v", id, err) + } + return intent +} + +// snDeliver acknowledges claim as delivered with plausible coordinates. +func snDeliver(t *testing.T, st *Store, claim situation.NotificationClaim, ts string, now time.Time) { + t.Helper() + as := "thread" + if claim.Intent.EffectClass == situationmodel.EffectRootSync { + as = "root" + } + if err := st.MarkNotificationDelivered(context.Background(), claim, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: ts, DeliveredAs: as}, now); err != nil { + t.Fatalf("MarkNotificationDelivered(%s): %v", claim.Intent.ID, err) + } +} + +// snClaimOne claims exactly one intent and fails the test when the claim +// returns anything other than one row. +func snClaimOne(t *testing.T, st *Store, now time.Time) situation.NotificationClaim { + t.Helper() + claims, err := st.ClaimNotificationIntents(context.Background(), snOwner, now, 5*time.Minute, 25) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + if len(claims) != 1 { + t.Fatalf("claimed %d intents, want exactly 1", len(claims)) + } + return claims[0] +} + +func snClasses(claims []situation.NotificationClaim) []string { + out := make([]string, 0, len(claims)) + for _, c := range claims { + out = append(out, string(c.Intent.EffectClass)) + } + return out +} + +// ---------------------------------------------------------------------- +// Step 1: claim ordering, fencing, heartbeat, release, recovery. +// ---------------------------------------------------------------------- + +// TestNotificationClaimFencesOwnerTokenAndLease proves the three fences: +// a claim is exclusive while its lease holds, the claim token increases +// monotonically on every (re)claim, and an expired lease is reclaimable by +// a different owner — whose new token fences the old holder out. +func TestNotificationClaimFencesOwnerTokenAndLease(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, _ = snSeedOneCycle(t, st, "group-claim-fence", now) + + first := snClaimOne(t, st, now) + if first.ClaimOwner != snOwner || first.ClaimToken < 1 { + t.Fatalf("first claim = %+v, want owner %q and a positive token", first, snOwner) + } + + // While the lease holds no other owner may take the same head. + others, err := st.ClaimNotificationIntents(ctx, "notify-b", now.Add(time.Second), 5*time.Minute, 25) + if err != nil { + t.Fatalf("second ClaimNotificationIntents: %v", err) + } + for _, c := range others { + if c.Intent.ID == first.Intent.ID { + t.Fatalf("intent %s was claimed twice under a live lease", c.Intent.ID) + } + } + + // After the lease expires the row is reclaimable, with a higher token. + expired := now.Add(6 * time.Minute) + reclaimed, err := st.ClaimNotificationIntents(ctx, "notify-b", expired, 5*time.Minute, 25) + if err != nil { + t.Fatalf("reclaim: %v", err) + } + if len(reclaimed) != 1 || reclaimed[0].Intent.ID != first.Intent.ID { + t.Fatalf("reclaimed = %v, want the one expired intent %s", snClasses(reclaimed), first.Intent.ID) + } + if reclaimed[0].ClaimToken <= first.ClaimToken { + t.Fatalf("reclaimed token %d must exceed the expired holder's %d", reclaimed[0].ClaimToken, first.ClaimToken) + } + + // The original holder is now fenced out of every acknowledgement. + if err := st.MarkNotificationDelivered(ctx, first, + situation.NotificationDelivery{Channel: "C", MessageTS: "1.1", DeliveredAs: "root"}, expired); !errors.Is(err, ErrNotificationClaimLost) { + t.Fatalf("stale delivered ack = %v, want ErrNotificationClaimLost", err) + } + if got := snIntent(t, st, first.Intent.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("intent status after a stale ack = %q, want unchanged pending", got.Status) + } +} + +// TestNotificationClaimOrdersRootBeforeJournalThenBySequence pins the +// per-Situation queue: the coalescible root projection first, then the +// immutable journal in Transition-sequence order, exactly one head at a +// time so a later entry can never pass an earlier one. +func TestNotificationClaimOrdersRootBeforeJournalThenBySequence(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-claim-order", now) + _ = snCommit(t, st, sitID, &first, now.Add(time.Minute)) + + // The head is the current root projection, even though sequence 1's + // journal entry is older. + root := snClaimOne(t, st, now.Add(2*time.Minute)) + if root.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("head effect class = %q, want root_sync", root.Intent.EffectClass) + } + snDeliver(t, st, root, "100.1", now.Add(2*time.Minute)) + + // Now the immutable journal drains in sequence order, one at a time. + seen := []int{} + for i := 0; i < 4; i++ { + at := now.Add(time.Duration(3+i) * time.Minute) + claims, err := st.ClaimNotificationIntents(context.Background(), snOwner, at, 5*time.Minute, 25) + if err != nil { + t.Fatalf("claim round %d: %v", i, err) + } + if len(claims) == 0 { + break + } + if len(claims) != 1 { + t.Fatalf("claim round %d returned %d intents, want at most the one Situation head", i, len(claims)) + } + c := claims[0] + if c.Intent.TransitionSequence == nil { + t.Fatalf("journal claim %s has no transition sequence", c.Intent.ID) + } + seen = append(seen, *c.Intent.TransitionSequence) + snDeliver(t, st, c, "20"+string(rune('0'+i))+".1", at) + } + for i := 1; i < len(seen); i++ { + if seen[i] < seen[i-1] { + t.Fatalf("journal delivered out of Transition-sequence order: %v", seen) + } + } + if len(seen) < 2 { + t.Fatalf("expected at least both journal entries to drain, got %v", seen) + } +} + +// TestNotificationClaimWaitsForDurableRoot proves a reply is not claimable +// — and so consumes no delivery attempt — until the Situation's root +// coordinates are durably published. +func TestNotificationClaimWaitsForDurableRoot(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, commit := snSeedOneCycle(t, st, "group-claim-root-dep", now) + thread := shIntentOfClass(t, commit.History.Intents, situationmodel.EffectThreadAppend) + + // Block the root out of the queue so the reply is the only candidate. + root := snClaimOne(t, st, now) + if root.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("head = %q, want root_sync", root.Intent.EffectClass) + } + if err := st.RetryNotificationIntent(ctx, root, "ratelimited", now.Add(time.Hour)); err != nil { + t.Fatalf("RetryNotificationIntent: %v", err) + } + + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(time.Minute), 5*time.Minute, 25) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + if len(claims) != 0 { + t.Fatalf("claimed %v while the root is undelivered, want nothing", snClasses(claims)) + } + if got := snIntent(t, st, thread.ID); got.AttemptCount != 0 { + t.Fatalf("root-dependent reply attempt_count = %d, want 0 (it never became claimable)", got.AttemptCount) + } + if _, _, ok, err := st.GetSituationRootCoordinates(ctx, sitID); err != nil || ok { + t.Fatalf("GetSituationRootCoordinates before delivery = (ok=%v, err=%v), want (false, nil)", ok, err) + } +} + +// TestNotificationClaimHonorsRetryScheduleWithoutReordering proves a later +// journal entry cannot pass an earlier pending one that is merely waiting +// out its retry delay. +func TestNotificationClaimHonorsRetryScheduleWithoutReordering(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-claim-retry-order", now) + second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) + + root := snClaimOne(t, st, now.Add(2*time.Minute)) + snDeliver(t, st, root, "100.1", now.Add(2*time.Minute)) + + earlier := snClaimOne(t, st, now.Add(3*time.Minute)) + if got := *earlier.Intent.TransitionSequence; got != 1 { + t.Fatalf("first journal head sequence = %d, want 1", got) + } + if err := st.RetryNotificationIntent(ctx, earlier, "ratelimited", now.Add(time.Hour)); err != nil { + t.Fatalf("RetryNotificationIntent: %v", err) + } + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(4*time.Minute), 5*time.Minute, 25) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + if len(claims) != 0 { + t.Fatalf("claimed %v while sequence 1 waits out its retry, want nothing", snClasses(claims)) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND status = 'pending'`, sitID); n < 2 { + t.Fatalf("pending intents = %d, want the held-back journal entries of both cycles", n) + } + _ = second +} + +// TestNotificationClaimRespectsBatchLimit proves the limit bounds one claim +// round across Situations. +func TestNotificationClaimRespectsBatchLimit(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + for i := 0; i < 3; i++ { + snSeedOneCycle(t, st, "group-claim-batch-"+string(rune('a'+i)), now) + } + claims, err := st.ClaimNotificationIntents(context.Background(), snOwner, now, 5*time.Minute, 2) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + if len(claims) != 2 { + t.Fatalf("claimed %d intents with limit 2, want 2", len(claims)) + } +} + +// TestNotificationClaimRecoversAbandonedClaims proves a crashed worker's +// expired lease is swept back to unclaimed and is then reclaimable. +func TestNotificationClaimRecoversAbandonedClaims(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, _ = snSeedOneCycle(t, st, "group-claim-abandoned", now) + + claim := snClaimOne(t, st, now) + if n, err := st.RecoverExpiredNotificationClaims(ctx, now.Add(time.Minute)); err != nil || n != 0 { + t.Fatalf("RecoverExpiredNotificationClaims before expiry = (%d, %v), want (0, nil)", n, err) + } + n, err := st.RecoverExpiredNotificationClaims(ctx, now.Add(6*time.Minute)) + if err != nil || n != 1 { + t.Fatalf("RecoverExpiredNotificationClaims after expiry = (%d, %v), want (1, nil)", n, err) + } + recovered := snIntent(t, st, claim.Intent.ID) + if recovered.ClaimOwner != nil || recovered.LeaseExpiresAt != nil { + t.Fatalf("recovered intent still holds a claim: %+v", recovered) + } + if recovered.Status != situationmodel.IntentPending { + t.Fatalf("recovered intent status = %q, want pending", recovered.Status) + } + if err := st.HeartbeatNotificationClaim(ctx, claim, now.Add(7*time.Minute), 5*time.Minute); !errors.Is(err, ErrNotificationClaimLost) { + t.Fatalf("heartbeat after recovery = %v, want ErrNotificationClaimLost", err) + } +} + +// ---------------------------------------------------------------------- +// Step 1: acknowledgement transitions. +// ---------------------------------------------------------------------- + +// TestNotificationAckDeliveredWritesRootCoordinates proves root coordinates +// land only when the matching fenced root delivery is acknowledged, and +// that the acknowledgement unblocks the Situation's dependent history. +func TestNotificationAckDeliveredWritesRootCoordinates(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := snSeedOneCycle(t, st, "group-ack-root", now) + + root := snClaimOne(t, st, now) + snDeliver(t, st, root, "100.1", now) + + channel, ts, ok, err := st.GetSituationRootCoordinates(ctx, sitID) + if err != nil || !ok || channel != "C-sit" || ts != "100.1" { + t.Fatalf("root coordinates = (%q,%q,%v,%v), want (C-sit,100.1,true,nil)", channel, ts, ok, err) + } + stored := snIntent(t, st, root.Intent.ID) + if stored.Status != situationmodel.IntentDelivered || stored.DeliveredAt == nil || + stored.Channel == nil || stored.MessageTS == nil || stored.DeliveredAs == nil { + t.Fatalf("delivered intent = %+v, want a complete delivered record", stored) + } + if stored.ClaimOwner != nil || stored.LeaseExpiresAt != nil { + t.Fatalf("delivered intent still holds a claim: %+v", stored) + } + next := snClaimOne(t, st, now.Add(time.Second)) + if next.Intent.EffectClass != situationmodel.EffectThreadAppend { + t.Fatalf("next claim = %q, want the now-unblocked thread_append", next.Intent.EffectClass) + } +} + +// TestNotificationAckStaleAcknowledgementChangesZeroRows walks every +// acknowledgement with a claim whose token has moved on and proves each one +// changes nothing. +func TestNotificationAckStaleAcknowledgementChangesZeroRows(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, _ = snSeedOneCycle(t, st, "group-ack-stale", now) + + claim := snClaimOne(t, st, now) + stale := claim + stale.ClaimToken = claim.ClaimToken + 41 + + acks := map[string]func() error{ + "delivered": func() error { + return st.MarkNotificationDelivered(ctx, stale, + situation.NotificationDelivery{Channel: "C", MessageTS: "9.9", DeliveredAs: "root"}, now) + }, + "retry": func() error { return st.RetryNotificationIntent(ctx, stale, "ratelimited", now.Add(time.Minute)) }, + "blocked": func() error { return st.BlockNotificationConfiguration(ctx, stale, "invalid_auth", now) }, + "failed": func() error { return st.FailNotificationIntent(ctx, stale, "invalid_payload", now) }, + "heartbeat": func() error { return st.HeartbeatNotificationClaim(ctx, stale, now, 5*time.Minute) }, + "release": func() error { return st.ReleaseNotificationClaim(ctx, stale, now) }, + } + before := snIntent(t, st, claim.Intent.ID) + for name, ack := range acks { + if err := ack(); !errors.Is(err, ErrNotificationClaimLost) { + t.Fatalf("stale %s ack = %v, want ErrNotificationClaimLost", name, err) + } + after := snIntent(t, st, claim.Intent.ID) + if after.Status != before.Status || after.AttemptCount != before.AttemptCount || + after.ClaimToken != before.ClaimToken { + t.Fatalf("stale %s ack changed durable state: %+v -> %+v", name, before, after) + } + } +} + +// TestNotificationAckHeartbeatAndReleaseKeepTheIntentClaimable proves a +// heartbeat moves the lease deadline without changing status, and a clean +// release hands the row straight back. +func TestNotificationAckHeartbeatAndReleaseKeepTheIntentClaimable(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, _ = snSeedOneCycle(t, st, "group-ack-heartbeat", now) + + claim := snClaimOne(t, st, now) + if err := st.HeartbeatNotificationClaim(ctx, claim, now.Add(30*time.Second), 5*time.Minute); err != nil { + t.Fatalf("HeartbeatNotificationClaim: %v", err) + } + beat := snIntent(t, st, claim.Intent.ID) + if beat.Status != situationmodel.IntentPending || beat.LeaseExpiresAt == nil || + !beat.LeaseExpiresAt.After(now.Add(5*time.Minute)) { + t.Fatalf("heartbeat left %+v, want pending with an extended lease", beat) + } + if err := st.ReleaseNotificationClaim(ctx, claim, now.Add(time.Minute)); err != nil { + t.Fatalf("ReleaseNotificationClaim: %v", err) + } + released := snIntent(t, st, claim.Intent.ID) + if released.ClaimOwner != nil || released.LeaseExpiresAt != nil || released.Status != situationmodel.IntentPending { + t.Fatalf("released intent = %+v, want pending and unclaimed", released) + } + again := snClaimOne(t, st, now.Add(2*time.Minute)) + if again.Intent.ID != claim.Intent.ID { + t.Fatalf("reclaimed %s, want the released %s", again.Intent.ID, claim.Intent.ID) + } +} + +// TestNotificationAckRetryBlockAndFailPreserveAttempts pins the three +// non-delivery outcomes: retry keeps the intent pending with a due time, +// configuration blocking is durable with no retry time, and failure is +// terminal-until-redriven. None of them ever resets the attempt count. +func TestNotificationAckRetryBlockAndFailPreserveAttempts(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, _ = snSeedOneCycle(t, st, "group-ack-outcomes", now) + + first := snClaimOne(t, st, now) + if first.Intent.AttemptCount != 1 { + t.Fatalf("first claim attempt_count = %d, want 1", first.Intent.AttemptCount) + } + if err := st.RetryNotificationIntent(ctx, first, "ratelimited", now.Add(5*time.Second)); err != nil { + t.Fatalf("RetryNotificationIntent: %v", err) + } + retried := snIntent(t, st, first.Intent.ID) + if retried.Status != situationmodel.IntentPending || retried.RetryAt == nil || retried.AttemptCount != 1 { + t.Fatalf("retried intent = %+v, want pending with retry_at and attempt_count 1", retried) + } + if retried.LastErrorClass == nil || *retried.LastErrorClass != "ratelimited" { + t.Fatalf("retried last_error_class = %v, want ratelimited", retried.LastErrorClass) + } + + second := snClaimOne(t, st, now.Add(10*time.Second)) + if second.Intent.AttemptCount != 2 { + t.Fatalf("second claim attempt_count = %d, want 2", second.Intent.AttemptCount) + } + if err := st.BlockNotificationConfiguration(ctx, second, "invalid_auth", now.Add(10*time.Second)); err != nil { + t.Fatalf("BlockNotificationConfiguration: %v", err) + } + blocked := snIntent(t, st, second.Intent.ID) + if blocked.Status != situationmodel.IntentBlockedConfiguration || blocked.AttemptCount != 2 || + blocked.RetryAt != nil || blocked.ClaimOwner != nil { + t.Fatalf("blocked intent = %+v, want durable blocked_configuration keeping attempts", blocked) + } + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(time.Hour), 5*time.Minute, 25) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + for _, c := range claims { + if c.Intent.ID == second.Intent.ID { + t.Fatal("a blocked_configuration intent must not be claimable") + } + } + + // Reactivate, then prove an invalid payload fails terminally. + if n, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(time.Hour)); err != nil || n != 1 { + t.Fatalf("ReactivateConfigurationBlocked = (%d, %v), want (1, nil)", n, err) + } + third := snClaimOne(t, st, now.Add(2*time.Hour)) + if third.Intent.AttemptCount != 3 { + t.Fatalf("post-reactivation attempt_count = %d, want the preserved 3", third.Intent.AttemptCount) + } + if err := st.FailNotificationIntent(ctx, third, "invalid_payload", now.Add(2*time.Hour)); err != nil { + t.Fatalf("FailNotificationIntent: %v", err) + } + failed := snIntent(t, st, third.Intent.ID) + if failed.Status != situationmodel.IntentFailed || failed.RetryAt != nil || failed.AttemptCount != 3 { + t.Fatalf("failed intent = %+v, want failed with no retry time and attempts preserved", failed) + } +} + +// TestNotificationAckRedrivenRootReleasesDependentHistory pins the spec's +// "a failed root never causes later effects to dead-letter in a chain": +// the dependents wait, and become claimable once the same root is explicitly +// redriven and delivered. +func TestNotificationAckRedrivenRootReleasesDependentHistory(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, commit := snSeedOneCycle(t, st, "group-ack-redrive", now) + thread := shIntentOfClass(t, commit.History.Intents, situationmodel.EffectThreadAppend) + + root := snClaimOne(t, st, now) + if err := st.FailNotificationIntent(ctx, root, "invalid_payload", now); err != nil { + t.Fatalf("FailNotificationIntent: %v", err) + } + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(time.Minute), 5*time.Minute, 25) + if err != nil { + t.Fatalf("ClaimNotificationIntents: %v", err) + } + if len(claims) != 0 { + t.Fatalf("claimed %v behind a failed root, want nothing (dependents wait, never dead-letter)", snClasses(claims)) + } + if got := snIntent(t, st, thread.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("dependent journal entry status = %q, want an untouched pending", got.Status) + } + + if err := st.RedriveFailedNotificationIntent(ctx, root.Intent.ID, now.Add(2*time.Minute)); err != nil { + t.Fatalf("RedriveFailedNotificationIntent: %v", err) + } + redriven := snClaimOne(t, st, now.Add(3*time.Minute)) + if redriven.Intent.ID != root.Intent.ID || redriven.Intent.AttemptCount != 2 { + t.Fatalf("redriven claim = %+v, want the same root with its attempts preserved", redriven.Intent) + } + snDeliver(t, st, redriven, "100.1", now.Add(3*time.Minute)) + + released := snClaimOne(t, st, now.Add(4*time.Minute)) + if released.Intent.ID != thread.ID { + t.Fatalf("post-redrive claim = %s, want the previously blocked journal entry %s", released.Intent.ID, thread.ID) + } +} + +// ---------------------------------------------------------------------- +// Step 2: supersession and revalidation. +// ---------------------------------------------------------------------- + +// TestNotificationSupersessionTakesTheClaimFromAnInFlightRootSync is the +// Task 5 handoff (R4): a concurrent authoritative commit may supersede the +// exact root projection this worker is mid-flight on. The acknowledgement +// must be refused, distinguishably, without corrupting anything. +func TestNotificationSupersessionTakesTheClaimFromAnInFlightRootSync(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-supersede-inflight", now) + + claim := snClaimOne(t, st, now) + if claim.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("head = %q, want root_sync", claim.Intent.EffectClass) + } + + // A concurrent controller commit supersedes exactly this projection. + second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) + + superseded := snIntent(t, st, claim.Intent.ID) + if superseded.Status != situationmodel.IntentSuperseded { + t.Fatalf("in-flight root status = %q, want superseded", superseded.Status) + } + if superseded.ClaimOwner != nil || superseded.LeaseExpiresAt != nil { + t.Fatalf("superseded root still carries a claim: %+v", superseded) + } + + err := st.MarkNotificationDelivered(ctx, claim, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: "100.1", DeliveredAs: "root"}, now.Add(2*time.Minute)) + if !errors.Is(err, ErrNotificationIntentSuperseded) { + t.Fatalf("delivered ack on a superseded root = %v, want ErrNotificationIntentSuperseded", err) + } + after := snIntent(t, st, claim.Intent.ID) + if after.Status != situationmodel.IntentSuperseded || after.DeliveredAt != nil { + t.Fatalf("superseded root after a refused ack = %+v, want it untouched", after) + } + if _, _, ok, err := st.GetSituationRootCoordinates(ctx, sitID); err != nil || ok { + t.Fatalf("root coordinates after a refused ack = (ok=%v, err=%v), want (false, nil)", ok, err) + } + + // The replacement projection is the claimable head, and its own + // delivery still works normally. + next := snClaimOne(t, st, now.Add(3*time.Minute)) + replacement := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) + if next.Intent.ID != replacement.ID { + t.Fatalf("next head = %s, want the replacement root %s", next.Intent.ID, replacement.ID) + } + snDeliver(t, st, next, "101.1", now.Add(3*time.Minute)) +} + +// TestNotificationSupersessionLeavesImmutableEntriesAndOtherEpisodes proves +// a newer root projection coalesces only the older root of its own +// Situation: immutable journal entries and other Situations' pending work +// are untouched. +func TestNotificationSupersessionLeavesImmutableEntriesAndOtherEpisodes(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-supersede-scope", now) + otherID, otherCommit := snSeedOneCycle(t, st, "group-supersede-other", now) + + firstThread := shIntentOfClass(t, first.History.Intents, situationmodel.EffectThreadAppend) + otherRoot := shIntentOfClass(t, otherCommit.History.Intents, situationmodel.EffectRootSync) + + _ = snCommit(t, st, sitID, &first, now.Add(time.Minute)) + + if got := snIntent(t, st, firstThread.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("immutable journal entry status = %q, want pending (never superseded)", got.Status) + } + if got := snIntent(t, st, otherRoot.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("other Situation's root status = %q, want an untouched pending", got.Status) + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE status = 'superseded'`); n != 1 { + t.Fatalf("superseded rows = %d, want exactly the one replaced root projection", n) + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND status = 'superseded'`, otherID); n != 0 { + t.Fatalf("other Situation had %d superseded intents, want 0", n) + } +} + +// TestNotificationSupersessionRecordsDelayedThreadDelivery pins the durable +// half of the stale-handoff rule: a revalidated-stale broadcast is recorded +// as a delayed_thread delivery, never as a broadcast. +func TestNotificationSupersessionRecordsDelayedThreadDelivery(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + _, commit := snSeedOneCycle(t, st, "group-supersede-delayed", now) + thread := shIntentOfClass(t, commit.History.Intents, situationmodel.EffectThreadAppend) + + root := snClaimOne(t, st, now) + snDeliver(t, st, root, "100.1", now) + entry := snClaimOne(t, st, now.Add(time.Second)) + if entry.Intent.ID != thread.ID { + t.Fatalf("claimed %s, want the journal entry %s", entry.Intent.ID, thread.ID) + } + if err := st.MarkNotificationDelivered(ctx, entry, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: "100.2", DeliveredAs: "delayed_thread"}, + now.Add(time.Second)); err != nil { + t.Fatalf("MarkNotificationDelivered(delayed_thread): %v", err) + } + stored := snIntent(t, st, thread.ID) + if stored.DeliveredAs == nil || *stored.DeliveredAs != "delayed_thread" { + t.Fatalf("delivered_as = %v, want delayed_thread", stored.DeliveredAs) + } + // A journal delivery never rewrites the Situation's root coordinates. + _, ts, ok, err := st.GetSituationRootCoordinates(ctx, *stored.SituationID) + if err != nil || !ok || ts != "100.1" { + t.Fatalf("root coordinates = (%q,%v,%v), want the root's own 100.1", ts, ok, err) + } +} From 3ab73d66f4eb01a84afd839e7b40d92a4cfab117 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 05:13:32 +0300 Subject: [PATCH 11/31] fix(situation): deliver a poked Transition once, as the broadcast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Transition selected as the main-channel poke was getting BOTH a thread_append and a broadcast_handoff. Migration 0018's uniqueness is (situation_id, transition_sequence, effect_class), so both were insertable, and Task 6 renders both classes from the same stored journal data — every handoff/escalation posted the identical journal entry to Slack twice, where spec describes exactly one broadcast reply. PlanNotificationIntents now selects the poke before building the reply loop and emits exactly one reply per journaled Transition: broadcast_handoff for the poked one, thread_append for every other, both in Transition-sequence order across the combined set. The one deliberate exception stays: a poke below the operator's Slack floor keeps its withheld broadcast_handoff as a durable decision AND gets a quiet thread_append, since the floor "never suppresses a non-broadcast journal entry" — only one of the two is ever delivered. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/situation/notification_plan.go | 73 +++++++--- internal/situation/notification_plan_test.go | 142 ++++++++++++++++--- 2 files changed, 170 insertions(+), 45 deletions(-) diff --git a/internal/situation/notification_plan.go b/internal/situation/notification_plan.go index 5cbb332..2440129 100644 --- a/internal/situation/notification_plan.go +++ b/internal/situation/notification_plan.go @@ -51,19 +51,24 @@ type PublicationInput struct { // // - one coalescible `root_sync` carrying the current Episode-summary // version and the committed contract deadline it renders (R4); -// - one immutable `thread_append` per journaled Transition, in sequence -// order, each rendering only its own Transition's stored journal data; -// - at most one `broadcast_handoff` — the single new main-channel poke a -// commit may create, when the permitted poke class allows it, the -// repage cooldown has elapsed for the one class it gates, and the root -// is already published (an unpublished root's first post IS the poke); +// - exactly one immutable reply per journaled Transition, in sequence +// order, each rendering only its own Transition's stored journal data: +// `thread_append` for a quiet entry, or `broadcast_handoff` for the one +// Transition (at most) that may create a new main-channel poke — when +// the permitted poke class allows it, the repage cooldown has elapsed +// for the one class it gates, and the root is already published (an +// unpublished root's first post IS the poke). The two classes render +// the same journal data, so a poked Transition never also gets a +// thread entry: that would post it to Slack twice; // - on a non-material cycle, only the R4 deadline refresh, and only when // the root is published, its last delivered promise has passed, and // this commit carries a different deadline. // // A poke below the operator's Slack floor becomes a durable // `withheld_by_operator_slack_floor` decision, never an absent row, and the -// floor never suppresses a non-broadcast journal entry. +// floor never suppresses a non-broadcast journal entry — a withheld +// broadcast is therefore the one case where a Transition carries two +// intents, of which only the quiet thread entry is ever delivered. func PlanNotificationIntents(in PublicationInput) ([]model.NotificationIntent, error) { if err := validatePublicationInput(in); err != nil { return nil, err @@ -93,29 +98,51 @@ func PlanNotificationIntents(in PublicationInput) ([]model.NotificationIntent, e } out = append(out, root) + // At most one new main-channel poke per commit, and none at all when + // the root post above already is one. Deciding this BEFORE the journal + // loop is what keeps a poked Transition from being delivered twice. + pokeSequence := 0 + if !rootPoke { + if poke, ok := selectPoke(in); ok { + pokeSequence = poke.Sequence + } + } + // Immutable journal entries, one per journaled Transition, in sequence - // order. These are never pokes and never carry a summary version. + // order. Each Transition produces exactly ONE reply: the poked one is + // broadcast (a handoff edits the root and then creates one broadcast + // reply), every other one is a quiet thread entry. Both classes render + // the same stored journal data, so emitting both for one Transition + // would post it to Slack twice. for _, tr := range in.Transitions { - if tr.JournalKind == model.JournalNone { + poked := pokeSequence != 0 && tr.Sequence == pokeSequence + if tr.JournalKind == model.JournalNone && !poked { + continue + } + if !poked { + out = append(out, newIntent(in, model.EffectThreadAppend, tr, + threadKey(model.EffectThreadAppend, in.Situation.ID, tr.Sequence))) continue } - out = append(out, newIntent(in, model.EffectThreadAppend, tr, threadKey(model.EffectThreadAppend, in.Situation.ID, tr.Sequence))) - } - // At most one new main-channel poke per commit, and none at all when - // the root post above already is one. - if !rootPoke { - if poke, ok := selectPoke(in); ok { - priority := DeriveInterruptionPriority(poke) - broadcast := newIntent(in, model.EffectBroadcastHandoff, poke, - threadKey(model.EffectBroadcastHandoff, in.Situation.ID, poke.Sequence)) - broadcast.MainChannelPoke = true - broadcast.InterruptionPriority = &priority - if !MeetsSlackFloor(priority, in.SlackFloor) { - broadcast.Status = model.IntentWithheld - } + priority := DeriveInterruptionPriority(tr) + broadcast := newIntent(in, model.EffectBroadcastHandoff, tr, + threadKey(model.EffectBroadcastHandoff, in.Situation.ID, tr.Sequence)) + broadcast.MainChannelPoke = true + broadcast.InterruptionPriority = &priority + if MeetsSlackFloor(priority, in.SlackFloor) { out = append(out, broadcast) + continue } + // Below the operator's floor the poke is withheld as a durable + // decision, never an absent row — but the floor "never suppresses + // ... a non-broadcast journal entry", so the same Transition still + // gets its quiet thread entry. Only one of the two is ever + // delivered, so this is not the duplicate the branch above avoids. + broadcast.Status = model.IntentWithheld + out = append(out, + newIntent(in, model.EffectThreadAppend, tr, threadKey(model.EffectThreadAppend, in.Situation.ID, tr.Sequence)), + broadcast) } for i := range out { diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go index d387fd1..75c6d5d 100644 --- a/internal/situation/notification_plan_test.go +++ b/internal/situation/notification_plan_test.go @@ -77,6 +77,20 @@ func hsIntentsOfClass(intents []model.NotificationIntent, class model.EffectClas return out } +// hsReplyIntents returns the in-thread journal replies in planned order, +// across BOTH reply classes: spec's "immutable journal replies deliver in +// Transition-sequence order" applies to the combined set, and exactly one +// reply exists per journaled Transition. +func hsReplyIntents(intents []model.NotificationIntent) []model.NotificationIntent { + out := []model.NotificationIntent{} + for _, i := range intents { + if i.EffectClass == model.EffectThreadAppend || i.EffectClass == model.EffectBroadcastHandoff { + out = append(out, i) + } + } + return out +} + // ---------------------------------------------------------------------- // Roots. // ---------------------------------------------------------------------- @@ -261,25 +275,35 @@ func TestPlanNotificationIntentsJournalAppendsInSequenceOrder(t *testing.T) { trs, sum := hsCommitOf(t, c) got := hsPlan(t, hsPub(c, trs, sum)) - threads := hsIntentsOfClass(got, model.EffectThreadAppend) - if len(threads) != len(trs) { - t.Fatalf("got %d thread entries, want one per journaled transition (%d)", len(threads), len(trs)) + // Exactly one reply per journaled Transition, across BOTH reply + // classes, in Transition-sequence order: the two artifacts as quiet + // thread entries, the escalating controller-state Transition as the one + // broadcast — never both for the same Transition. + replies := hsReplyIntents(got) + if len(replies) != len(trs) { + t.Fatalf("got %d journal replies, want exactly one per journaled transition (%d)", len(replies), len(trs)) + } + wantClasses := []model.EffectClass{ + model.EffectThreadAppend, model.EffectThreadAppend, model.EffectBroadcastHandoff, } - for i, intent := range threads { + for i, intent := range replies { if intent.TransitionID == nil || *intent.TransitionID != trs[i].ID { - t.Errorf("thread entry %d references %v, want %q", i, intent.TransitionID, trs[i].ID) + t.Errorf("reply %d references %v, want %q", i, intent.TransitionID, trs[i].ID) } if intent.TransitionSequence == nil || *intent.TransitionSequence != trs[i].Sequence { - t.Errorf("thread entry %d sequence = %v, want %d", i, intent.TransitionSequence, trs[i].Sequence) + t.Errorf("reply %d sequence = %v, want %d", i, intent.TransitionSequence, trs[i].Sequence) + } + if intent.EffectClass != wantClasses[i] { + t.Errorf("reply %d class = %q, want %q", i, intent.EffectClass, wantClasses[i]) } if !intent.RequiresRoot { - t.Errorf("thread entry %d must wait for durable root coordinates", i) + t.Errorf("reply %d must wait for durable root coordinates", i) } - if intent.MainChannelPoke { - t.Errorf("thread entry %d must never be a poke", i) + if intent.MainChannelPoke != (intent.EffectClass == model.EffectBroadcastHandoff) { + t.Errorf("reply %d poke flag %v does not match its class %q", i, intent.MainChannelPoke, intent.EffectClass) } if intent.SummaryVersion != nil { - t.Errorf("thread entry %d must render its own transition, not a summary version", i) + t.Errorf("reply %d must render its own transition, not a summary version", i) } } } @@ -290,14 +314,19 @@ func TestPlanNotificationIntentsOperatorHandoffBroadcast(t *testing.T) { trs, sum := hsCommitOf(t, c) got := hsPlan(t, hsPub(c, trs, sum)) - threads := hsIntentsOfClass(got, model.EffectThreadAppend) - if len(threads) != 1 { - t.Fatalf("got %d thread entries, want exactly one journal entry", len(threads)) - } broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff) if len(broadcasts) != 1 { t.Fatalf("got %d broadcast effects, want at most one (and exactly one here)", len(broadcasts)) } + // "A handoff edits the root first and then creates ONE broadcast reply" + // — the handoff Transition's journal entry is that broadcast, never a + // broadcast plus a duplicate quiet thread reply of the same content. + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 0 { + t.Fatalf("got %d thread entries alongside the broadcast, want 0 — the same journal data would post twice", len(threads)) + } + if replies := hsReplyIntents(got); len(replies) != len(trs) { + t.Fatalf("got %d journal replies for %d journaled transitions, want exactly one each", len(replies), len(trs)) + } if !broadcasts[0].MainChannelPoke { t.Error("the handoff broadcast must be the main-channel poke") } @@ -360,6 +389,20 @@ func TestPlanNotificationIntentsAtMostOneBroadcastPerCommit(t *testing.T) { if !broadcasts[0].MainChannelPoke || broadcasts[0].InterruptionPriority == nil { t.Error("the escalation broadcast must be a poke carrying its evaluated priority") } + // One reply per journaled Transition: the artifacts as quiet + // thread entries, the escalation as the broadcast — and never a + // thread entry duplicating the broadcast's own Transition. + threads := hsIntentsOfClass(got, model.EffectThreadAppend) + if len(threads) != len(tc.artifacts) { + t.Errorf("got %d thread entries, want one per artifact (%d)", len(threads), len(tc.artifacts)) + } + for _, thread := range threads { + if thread.TransitionSequence != nil && broadcasts[0].TransitionSequence != nil && + *thread.TransitionSequence == *broadcasts[0].TransitionSequence { + t.Errorf("transition sequence %d has both a thread entry and a broadcast; it would post twice", + *thread.TransitionSequence) + } + } // The broadcast must name the controller-state Transition (last // in the commit, per R1) — the same authority the root_sync // references — never an artifact Transition. @@ -381,23 +424,78 @@ func TestPlanNotificationIntentsAtMostOneBroadcastPerCommit(t *testing.T) { // escalation must produce the same poke with and without an operator // artifact journaled ahead of it in the same commit. func TestPlanNotificationIntentsEscalationSurvivesJournaledArtifacts(t *testing.T) { - escalate := func(t *testing.T, artifacts []OperatorArtifactInput) int { + escalate := func(t *testing.T, artifacts []OperatorArtifactInput) (broadcasts, threads, replies, journaled int) { t.Helper() c := hsNext(t) c.Situation.Attention = model.AttentionUrgent c.Assessment.Attention = model.AttentionUrgent c.OperatorArtifacts = artifacts trs, sum := hsCommitOf(t, c) - return len(hsIntentsOfClass(hsPlan(t, hsPub(c, trs, sum)), model.EffectBroadcastHandoff)) + got := hsPlan(t, hsPub(c, trs, sum)) + return len(hsIntentsOfClass(got, model.EffectBroadcastHandoff)), + len(hsIntentsOfClass(got, model.EffectThreadAppend)), + len(hsReplyIntents(got)), len(trs) } - without := escalate(t, nil) - with := escalate(t, []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))}) - if without != 1 { - t.Fatalf("urgent escalation alone produced %d broadcasts, want 1", without) + withoutB, withoutT, withoutR, withoutN := escalate(t, nil) + withB, withT, withR, withN := escalate(t, []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))}) + + if withoutB != 1 { + t.Fatalf("urgent escalation alone produced %d broadcasts, want 1", withoutB) } - if with != without { - t.Errorf("a journaled operator artifact changed the escalation poke: %d broadcasts with, %d without", with, without) + if withB != withoutB { + t.Errorf("a journaled operator artifact changed the escalation poke: %d broadcasts with, %d without", withB, withoutB) + } + if withoutT != 0 { + t.Errorf("the escalation alone produced %d thread entries, want 0 — the broadcast IS its journal entry", withoutT) + } + if withT != 1 { + t.Errorf("got %d thread entries, want exactly the artifact's own", withT) + } + if withoutR != withoutN || withR != withN { + t.Errorf("journal replies (%d for %d transitions, %d for %d) must be exactly one per journaled transition", + withoutR, withoutN, withR, withN) + } +} + +// TestPlanNotificationIntentsPokedTransitionIsBroadcastOnly is the direct +// regression: in a commit with two journaled Transitions where the second +// qualifies as a poke, the artifact gets a thread entry, the poked +// controller-state Transition gets a broadcast, and neither Transition ever +// gets both. Task 6 renders both classes from the same stored journal data, +// so a Transition carrying both would be posted to Slack twice. +func TestPlanNotificationIntentsPokedTransitionIsBroadcastOnly(t *testing.T) { + c := hsNext(t) + c.Assessment.ActionContract = hsOperatorContract(c.Now.Add(time.Minute)) + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("input-1", artifactKindAnnotation, hsNow(t))} + trs, sum := hsCommitOf(t, c) + if len(trs) != 2 { + t.Fatalf("fixture built %d transitions, want the artifact plus the handoff", len(trs)) + } + artifact, handoff := trs[0], trs[1] + + got := hsPlan(t, hsPub(c, trs, sum)) + replies := hsReplyIntents(got) + if len(replies) != 2 { + t.Fatalf("got %d journal replies, want exactly one per journaled transition: %+v", len(replies), replies) + } + if replies[0].EffectClass != model.EffectThreadAppend || *replies[0].TransitionID != artifact.ID { + t.Errorf("reply 0 = %q for %v, want a thread_append for the artifact %q", + replies[0].EffectClass, replies[0].TransitionID, artifact.ID) + } + if replies[1].EffectClass != model.EffectBroadcastHandoff || *replies[1].TransitionID != handoff.ID { + t.Errorf("reply 1 = %q for %v, want a broadcast_handoff for the handoff %q", + replies[1].EffectClass, replies[1].TransitionID, handoff.ID) + } + // Sequence order is preserved across the combined reply set. + if *replies[0].TransitionSequence >= *replies[1].TransitionSequence { + t.Errorf("replies are out of transition-sequence order: %d then %d", + *replies[0].TransitionSequence, *replies[1].TransitionSequence) + } + for _, intent := range hsIntentsOfClass(got, model.EffectThreadAppend) { + if *intent.TransitionID == handoff.ID { + t.Error("the poked transition also got a thread entry; its journal data would post twice") + } } } From f3412994d214afc0d6af2d92dc8ef2a0de7e29eb Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 05:34:18 +0300 Subject: [PATCH 12/31] test(store): pin that a floor-withheld effect is never claimable Task 4's 3ab73d6 stopped emitting a paired thread_append for a DELIVERED poke, leaving one shape where a Transition still carries two intents: a poke below the operator's Slack floor keeps a durable withheld broadcast_handoff AND the quiet thread_append the floor never suppresses. That shape reaches the claim query, so pin it there: the withheld decision is never handed out and consumes no attempt, and it never shadows the quiet entry at the same Transition sequence in its Situation's queue. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- .../store/situation_notifications_test.go | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index 11144ec..f5b42f0 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -48,6 +48,37 @@ func snCommit(t *testing.T, st *Store, sitID string, prior *situation.Controller return commit } +// snCommitBelowFloor is snCommit's second-cycle form with an operator Slack +// floor high enough to withhold the poke: the poked Transition then carries +// BOTH a durably withheld broadcast_handoff and the quiet thread_append the +// floor never suppresses (Task 4's `3ab73d6`). +func snCommitBelowFloor(t *testing.T, st *Store, sitID string, prior situation.ControllerCommit, + now time.Time) situation.ControllerCommit { + t.Helper() + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim := claimSituation(t, st, sitID, "controller-a", now) + cycle := shPrepare(t, claim, shOperatorContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + last := prior.History.Transitions[len(prior.History.Transitions)-1] + cycle.Change.PriorTransition = &last + cycle.Change.PriorSummary = prior.History.Summary + cycle.Publish.PriorTransition = &last + cycle.Publish.RootPublished = true + cycle.Publish.SlackFloor = situationmodel.InterruptionCritical + // The shared fixture anchors every conclusion on the deterministic + // critical floor, which always passes any floor. Swap in an ordinary + // Sufficient reason so this poke derives `high` and the operator's + // critical floor can actually withhold it. + concl := *cycle.Change.Projection.Assessment + concl.SufficientReasonCode = "duration_outlier" + cycle.Change.Projection.Assessment = &concl + commit := shDerive(t, cycle) + if err := st.CommitController(context.Background(), claim, commit); err != nil { + t.Fatalf("CommitController: %v", err) + } + return commit +} + // snSeedOneCycle creates a Situation with exactly one committed cycle: a // pending root_sync plus one immutable thread_append at sequence 1. func snSeedOneCycle(t *testing.T, st *Store, group string, now time.Time) (string, situation.ControllerCommit) { @@ -636,3 +667,61 @@ func TestNotificationSupersessionRecordsDelayedThreadDelivery(t *testing.T) { t.Fatalf("root coordinates = (%q,%v,%v), want the root's own 100.1", ts, ok, err) } } + +// TestNotificationClaimNeverClaimsAFloorWithheldEffect pins the one shape +// where a Transition still carries two intents after Task 4's `3ab73d6`: a +// poke below the operator's Slack floor keeps a durably withheld +// broadcast_handoff AND the quiet thread_append the floor never suppresses. +// The withheld decision must never become claimable — only the quiet entry +// is ever delivered — and the withheld row must not shadow it in its own +// Situation's queue. +func TestNotificationClaimNeverClaimsAFloorWithheldEffect(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-claim-withheld", now) + second := snCommitBelowFloor(t, st, sitID, first, now.Add(time.Minute)) + + withheld := shIntentOfClass(t, second.History.Intents, situationmodel.EffectBroadcastHandoff) + if withheld.Status != situationmodel.IntentWithheld { + t.Fatalf("broadcast below the Slack floor has status %q, want withheld_by_operator_slack_floor", withheld.Status) + } + quiet := shIntentOfClass(t, second.History.Intents, situationmodel.EffectThreadAppend) + if quiet.TransitionSequence == nil || withheld.TransitionSequence == nil || + *quiet.TransitionSequence != *withheld.TransitionSequence { + t.Fatalf("the withheld broadcast and its quiet entry must share one Transition sequence: %v vs %v", + withheld.TransitionSequence, quiet.TransitionSequence) + } + + // Drain the Situation, proving the withheld row is never handed out and + // never blocks the queue behind it. + claimed := []string{} + for round := 0; round < 6; round++ { + at := now.Add(time.Duration(2+round) * time.Minute) + claims, err := st.ClaimNotificationIntents(context.Background(), snOwner, at, 5*time.Minute, 25) + if err != nil { + t.Fatalf("claim round %d: %v", round, err) + } + if len(claims) == 0 { + break + } + for i, c := range claims { + if c.Intent.ID == withheld.ID { + t.Fatal("a withheld_by_operator_slack_floor intent was claimed") + } + claimed = append(claimed, c.Intent.ID) + snDeliver(t, st, c, "30"+string(rune('0'+round))+"."+string(rune('0'+i)), at) + } + } + var sawQuiet bool + for _, id := range claimed { + if id == quiet.ID { + sawQuiet = true + } + } + if !sawQuiet { + t.Fatalf("claimed %v, want the quiet journal entry %s to have been delivered", claimed, quiet.ID) + } + if got := snIntent(t, st, withheld.ID); got.Status != situationmodel.IntentWithheld || got.AttemptCount != 0 { + t.Fatalf("withheld intent = %+v, want an untouched durable decision with no attempts", got) + } +} From ce4698c8df4a6ce59292ae20c0472047532a3513 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 06:17:54 +0300 Subject: [PATCH 13/31] fix(situation): keep one pending root, order replay by sequence, reactivate once Three review findings on the notification delivery worker. Reactivation deadlocked itself against the pending-root_sync unique index. It returned every blocked_configuration row to pending in one statement, but supersession only ever retires a PENDING root_sync, so a Situation can hold a blocked root beside a newer pending one. Reactivating the blocked one then violated notification_intents_root_sync_pending_idx, rolled back the whole configuration-generation advance, and left NOTHING reactivated for any Situation, permanently. Root projections are now reactivated per Situation: the newest live projection keeps (or takes) the single pending slot, older ones are coalesced into it through Task 5's own supersedePendingRootSyncTx, and when the newest is already pending the older blocked ones stay blocked -- migration 0018 calls a blocked root a resolved outcome, and the pending projection renders the same current state, so nothing is stranded. RedriveFailedNotificationIntent shared that hazard and now shares the fix: it coalesces an older pending root into the redriven one, and refuses with ErrNewerRootProjectionPending behind a newer one instead of aborting on the index. Claim ordering ranked effect class ABOVE Transition sequence, so every thread_append sorted ahead of every broadcast_handoff and an older poke could deliver after newer quiet entries -- against spec's 'immutable journal replies deliver in Transition-sequence order'. Class now enters the order in one place only (the root projection precedes its replies, which is the only class ordering the spec requires) and otherwise decides ties within one sequence. Reactivation fired on any successful Probe. Probe is only auth.test, so a valid token with a misconfigured channel succeeded forever: reactivate, claim, channel_not_found, block, reactivate -- unbounded generation and attempt growth, with first_failure_at reset every cycle so a real five-minute Delivery gap could never open. It is now once per process, on the startup probe, and exported as ReactivateConfiguration so Task 9's startup sequence can drive it explicitly instead. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- internal/situation/notification_worker.go | 68 +++++- .../situation/notification_worker_test.go | 74 ++++++ internal/store/notification_gaps.go | 180 ++++++++++++++- internal/store/situation_notifications.go | 134 +++++++---- .../store/situation_notifications_test.go | 218 ++++++++++++++++++ 5 files changed, 612 insertions(+), 62 deletions(-) diff --git a/internal/situation/notification_worker.go b/internal/situation/notification_worker.go index 7cad036..3657814 100644 --- a/internal/situation/notification_worker.go +++ b/internal/situation/notification_worker.go @@ -388,6 +388,9 @@ type NotificationWorker struct { startOnce sync.Once stopOnce sync.Once started atomic.Bool + // configurationReactivated is this PROCESS's one-shot guard for the + // startup configuration reactivation. See ReactivateConfiguration. + configurationReactivated atomic.Bool mu sync.Mutex // inflight holds every claim this worker currently owns, so Stop can @@ -518,9 +521,13 @@ func (w *NotificationWorker) shouldProbe(state SlackDeliveryState, now time.Time return true } // A REPLAYING generation is a recovered one: its own deliveries prove - // Slack health, so only an outage window, a still-open generation, or - // durably blocked configuration keeps probing. - if state.FirstFailureAt == nil && state.OpenGapStatus != "open" && state.BlockedConfigurationCount == 0 { + // Slack health. Durably blocked configuration justifies probing only + // until this process has applied its startup correction — after that a + // probe can no longer change the outcome (only a restart with corrected + // configuration can), and probing on it forever is the treadmill this + // worker must not run. + blockedStillMatters := state.BlockedConfigurationCount > 0 && !w.configurationReactivated.Load() + if state.FirstFailureAt == nil && state.OpenGapStatus != "open" && !blockedStillMatters { return false } wait := notificationRetryDelay(w.probeFailures, 0, w.cfg.RetryInitial, w.cfg.RetryMax, 0, 0) @@ -556,15 +563,8 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState if err := w.store.ObserveSlackSuccess(ctx, now); err != nil { w.logger.Error("situation: notification worker: record slack success failed", "err", err) } - if state.BlockedConfigurationCount > 0 { - n, err := w.store.ReactivateConfigurationBlocked(ctx, state.ConfigurationGeneration+1, now) - if err != nil { - w.logger.Error("situation: notification worker: reactivate configuration-blocked intents failed", "err", err) - } else if n > 0 { - w.count(func(s *NotificationWorkerStats) { s.Reactivated += int64(n) }) - w.logger.Info("situation: notification worker: slack configuration corrected; reactivated blocked intents", - "count", n, "configuration_generation", state.ConfigurationGeneration+1) - } + if _, err := w.ReactivateConfiguration(ctx); err != nil { + w.logger.Error("situation: notification worker: reactivate configuration-blocked intents failed", "err", err) } if generation, recovered, err := w.store.RecoverDeliveryGap(ctx, now); err != nil { w.logger.Error("situation: notification worker: recover delivery gap failed", "err", err) @@ -575,6 +575,50 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState } } +// ReactivateConfiguration applies corrected Slack configuration exactly ONCE +// per process: it advances the durable configuration generation and returns +// every eligible blocked_configuration intent to pending. It reports how +// many it reactivated, and (0, nil) once this process has already done it. +// +// Once per process, not once per successful probe, because Probe is only +// auth.test — it proves the TOKEN works and says nothing about the channel. +// With a valid token and a misconfigured channel, reactivating on every +// probe is a permanent loop: reactivate, claim, get channel_not_found, block, +// reactivate. That grows the configuration generation and every intent's +// attempt count without bound, and resets the continuous-failure window each +// cycle so a real Delivery gap can never open. spec.md ties this to startup — +// "Startup with corrected configuration increments a durable configuration +// generation" — and only a restart can actually change the configuration a +// blocked intent is blocked on. +// +// The worker calls this itself on its first successful probe (its startup +// probe). It is exported and idempotent so Task 9's startup sequence can +// instead drive it explicitly at step 5 of spec.md's startup order, before +// Receivers start; whichever runs first applies the correction. +func (w *NotificationWorker) ReactivateConfiguration(ctx context.Context) (int, error) { + if w.configurationReactivated.Swap(true) { + return 0, nil + } + state, err := w.store.GetSlackDeliveryState(ctx) + if err != nil { + return 0, fmt.Errorf("situation: notification worker: read slack delivery state: %w", err) + } + if state.BlockedConfigurationCount == 0 { + return 0, nil + } + generation := state.ConfigurationGeneration + 1 + n, err := w.store.ReactivateConfigurationBlocked(ctx, generation, w.now().UTC()) + if err != nil { + return 0, fmt.Errorf("situation: notification worker: reactivate configuration-blocked intents: %w", err) + } + if n > 0 { + w.count(func(s *NotificationWorkerStats) { s.Reactivated += int64(n) }) + w.logger.Info("situation: notification worker: slack configuration corrected; reactivated blocked intents", + "count", n, "configuration_generation", generation) + } + return n, nil +} + // observeFailure records one Slack failure against the continuous window and // emits the bounded WARNs the console action trail expects: one on the first // failure of a window, then paced retry WARNs — never one per attempt. diff --git a/internal/situation/notification_worker_test.go b/internal/situation/notification_worker_test.go index 8b73049..a5b4cb3 100644 --- a/internal/situation/notification_worker_test.go +++ b/internal/situation/notification_worker_test.go @@ -747,3 +747,77 @@ func TestNotificationWorkerStopReleasesHeldClaims(t *testing.T) { } }) } + +// TestNotificationWorkerReactivatesConfigurationOncePerProcess is the +// regression for reactivating on EVERY successful probe. Probe is only +// auth.test: with a valid token and a misconfigured CHANNEL it succeeds +// forever, so reactivating on it looped every poll — reactivate, claim, +// chat.postMessage fails channel_not_found, block, reactivate again. That +// grew configuration_generation and attempt_count without bound and reset +// first_failure_at every cycle, so a real five-minute Delivery gap could +// never open and the operator never got a recovery notice for what was, in +// effect, a permanent outage. +// +// The spec ties reactivation to STARTUP: "Successful startup probe +// reactivates configuration-blocked intents". +func TestNotificationWorkerReactivatesConfigurationOncePerProcess(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + // A permanently misconfigured channel: auth.test keeps succeeding and + // intents stay blocked, round after round. + store := &nwStore{state: SlackDeliveryState{ConfigurationGeneration: 3, BlockedConfigurationCount: 2}} + deliverer := &nwDeliverer{} + clock := &nwClock{at: now} + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", + Heartbeat: time.Hour, + Rand: func() float64 { return 0.5 }, + }, clock.now, nwLogger()) + + for round := 0; round < 5; round++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", round, err) + } + clock.advance(10 * time.Minute) // well past any probe backoff + } + store.snapshot(func(s *nwStore) { + if len(s.reactivated) != 1 || s.reactivated[0] != 4 { + t.Fatalf("reactivated with generations %v, want exactly one startup reactivation [4]", s.reactivated) + } + }) + if got := w.Stats().Reactivated; got != 1 { + t.Fatalf("Stats().Reactivated = %d, want 1", got) + } + // Once configuration has been applied for this process, durably blocked + // intents no longer justify probing — otherwise the worker probes every + // few seconds forever against a configuration only a restart can change. + if deliverer.probeCalls != 1 { + t.Fatalf("probe calls = %d, want the single startup probe", deliverer.probeCalls) + } +} + +// TestNotificationWorkerReactivateConfigurationIsIdempotentForStartup proves +// the explicit entry point Task 9's startup sequence calls is safe to invoke +// alongside the worker's own first probe: whichever runs first applies the +// correction, the other reports nothing to do. +func TestNotificationWorkerReactivateConfigurationIsIdempotentForStartup(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{state: SlackDeliveryState{ConfigurationGeneration: 7, BlockedConfigurationCount: 1}} + w := nwWorker(store, &nwDeliverer{}, now) + + n, err := w.ReactivateConfiguration(context.Background()) + if err != nil || n != 1 { + t.Fatalf("first ReactivateConfiguration = (%d, %v), want (1, nil)", n, err) + } + n, err = w.ReactivateConfiguration(context.Background()) + if err != nil || n != 0 { + t.Fatalf("second ReactivateConfiguration = (%d, %v), want (0, nil)", n, err) + } + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + store.snapshot(func(s *nwStore) { + if len(s.reactivated) != 1 || s.reactivated[0] != 8 { + t.Fatalf("reactivated with generations %v, want exactly [8]", s.reactivated) + } + }) +} diff --git a/internal/store/notification_gaps.go b/internal/store/notification_gaps.go index 73cda93..77adf0f 100644 --- a/internal/store/notification_gaps.go +++ b/internal/store/notification_gaps.go @@ -11,6 +11,7 @@ import ( "time" "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" ) // ---------------------------------------------------------------------- @@ -385,6 +386,13 @@ func (s *Store) CompleteDeliveryGap(ctx context.Context, now time.Time) (string, // The generation is a compare-and-set, not a blind write: a value that does // not advance the stored one reactivates nothing, so a restart loop can // never replay the same correction twice. +// +// Root projections need care (see reactivateBlockedRootSyncTx): returning +// every blocked row to pending in one statement violates migration 0018's +// single-pending-root_sync index the moment a Situation holds both a blocked +// root and a newer pending one, which would roll back the whole +// configuration-generation advance and strand EVERY Situation's +// reactivation, permanently. func (s *Store) ReactivateConfigurationBlocked(ctx context.Context, configurationGeneration int64, now time.Time) (int, error) { if configurationGeneration <= 0 { @@ -416,21 +424,14 @@ func (s *Store) ReactivateConfigurationBlocked(ctx context.Context, configuratio return 0, nil } - reactivated, err := tx.ExecContext(ctx, ` - UPDATE notification_intents - SET status = 'pending', retry_at = ?, claim_owner = NULL, lease_expires_at = NULL - WHERE status = 'blocked_configuration'`, nowStr) - if err != nil { - return 0, fmt.Errorf("store: reactivate configuration-blocked intents: %w", err) - } - n, err := reactivated.RowsAffected() + n, err := reactivateBlockedIntentsTx(ctx, tx, nowStr) if err != nil { - return 0, fmt.Errorf("store: count reactivated notification intents: %w", err) + return 0, err } if err := tx.Commit(); err != nil { return 0, fmt.Errorf("store: commit reactivate configuration-blocked intents: %w", err) } - return int(n), nil + return n, nil } // replayableBacklogTx counts the Situation-scoped delivery obligations @@ -447,3 +448,162 @@ func replayableBacklogTx(ctx context.Context, tx *sql.Tx, asOf string) (int, int } return affected, delayed, nil } + +// reactivateBlockedIntentsTx returns every eligible blocked_configuration +// intent to pending, due now, attempts preserved. +// +// It is split by effect class on purpose. thread_append, broadcast_handoff, +// and installation_gap_recovery are immutable one-per-subject effects with no +// pending-uniqueness constraint, so they reactivate in one statement. Root +// projections cannot: migration 0018's +// notification_intents_root_sync_pending_idx allows a Situation exactly ONE +// pending root_sync, and supersession only ever retires a PENDING one — so a +// Situation can legitimately hold a blocked root beside a newer pending root, +// and blindly reactivating the blocked one aborts the transaction. +func reactivateBlockedIntentsTx(ctx context.Context, tx *sql.Tx, nowStr string) (int, error) { + res, err := tx.ExecContext(ctx, ` + UPDATE notification_intents + SET status = 'pending', retry_at = ?, claim_owner = NULL, lease_expires_at = NULL + WHERE status = 'blocked_configuration' AND effect_class != 'root_sync'`, nowStr) + if err != nil { + return 0, fmt.Errorf("store: reactivate configuration-blocked effects: %w", err) + } + total, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count reactivated notification effects: %w", err) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT DISTINCT situation_id FROM notification_intents + WHERE status = 'blocked_configuration' AND effect_class = 'root_sync' AND situation_id IS NOT NULL + ORDER BY situation_id ASC`) + if err != nil { + return 0, fmt.Errorf("store: list situations with blocked root projections: %w", err) + } + situationIDs, err := scanStringRows(rows) + if err != nil { + return 0, fmt.Errorf("store: read situations with blocked root projections: %w", err) + } + for _, situationID := range situationIDs { + n, err := reactivateBlockedRootSyncTx(ctx, tx, situationID, nowStr) + if err != nil { + return 0, err + } + total += int64(n) + } + return int(total), nil +} + +// liveRootProjection is one root_sync still capable of holding its +// Situation's single pending-root slot: pending, or blocked on +// configuration. delivered/failed/withheld/superseded rows are resolved +// outcomes and never compete for it. +type liveRootProjection struct { + id string + status string +} + +// reactivateBlockedRootSyncTx restores exactly one pending root projection +// for situationID and reports how many blocked roots it reactivated (0 or 1). +// +// The newest live projection is the one corrected configuration should +// deliver — it renders current state, and every older one would render state +// already superseded by it. Two cases: +// +// - The newest is ALREADY pending. It holds the slot and says everything +// the older blocked ones would; they stay blocked_configuration, which +// migration 0018 explicitly calls a resolved outcome ("never one already +// delivered/blocked/failed/withheld ... not live candidates a newer +// root_sync coalesces away"). Nothing is stranded: the pending projection +// delivers the root coordinates every dependent effect waits on. +// +// - The newest is blocked. It is reactivated, and every older live +// projection is coalesced into it through Task 5's own +// supersedePendingRootSyncTx — the same supersession a newer commit +// performs. An older BLOCKED one reaches `superseded` the only way the +// schema permits, by being reactivated first: that is exactly what +// happened (corrected configuration returned it to pending) immediately +// followed by the newer projection coalescing it. +// +// The order is what keeps the unique index satisfied at every step: the +// pre-existing pending row is retired first, then each older blocked row is +// made pending and immediately coalesced, and only then does the keeper +// become pending. At no point do two root projections hold the slot. +func reactivateBlockedRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, nowStr string) (int, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT id, status FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' + AND status IN ('pending','blocked_configuration') + ORDER BY created_at ASC, id ASC`, situationID) + if err != nil { + return 0, fmt.Errorf("store: list live root projections for %s: %w", situationID, err) + } + live, err := scanLiveRootProjections(rows) + if err != nil { + return 0, err + } + if len(live) == 0 { + return 0, nil + } + keeper := live[len(live)-1] + if keeper.status == string(situationmodel.IntentPending) { + return 0, nil + } + + // Retire whichever projection currently holds the pending slot, if any. + if err := supersedePendingRootSyncTx(ctx, tx, situationID, keeper.id); err != nil { + return 0, err + } + for _, older := range live[:len(live)-1] { + if older.status != string(situationmodel.IntentBlockedConfiguration) { + continue // already retired by the supersession above + } + if err := setNotificationIntentPendingTx(ctx, tx, older.id, "blocked_configuration", nowStr); err != nil { + return 0, err + } + if err := supersedePendingRootSyncTx(ctx, tx, situationID, keeper.id); err != nil { + return 0, err + } + } + if err := setNotificationIntentPendingTx(ctx, tx, keeper.id, "blocked_configuration", nowStr); err != nil { + return 0, err + } + return 1, nil +} + +func scanLiveRootProjections(rows *sql.Rows) ([]liveRootProjection, error) { + defer func() { _ = rows.Close() }() + out := []liveRootProjection{} + for rows.Next() { + var p liveRootProjection + if err := rows.Scan(&p.id, &p.status); err != nil { + return nil, fmt.Errorf("store: scan live root projection: %w", err) + } + out = append(out, p) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate live root projections: %w", err) + } + return out, nil +} + +// setNotificationIntentPendingTx returns one intent in fromStatus to pending, +// due now, keeping its attempt count. It is fenced on the expected status so +// a row that moved on changes nothing. +func setNotificationIntentPendingTx(ctx context.Context, tx *sql.Tx, intentID, fromStatus, nowStr string) error { + res, err := tx.ExecContext(ctx, ` + UPDATE notification_intents + SET status = 'pending', retry_at = ?, claim_owner = NULL, lease_expires_at = NULL + WHERE id = ? AND status = ?`, nowStr, intentID, fromStatus) + if err != nil { + return fmt.Errorf("store: return notification intent %s to pending: %w", intentID, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("store: count notification intent %s returned to pending: %w", intentID, err) + } + if n != 1 { + return fmt.Errorf("store: notification intent %s is no longer %s", intentID, fromStatus) + } + return nil +} diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go index 516f772..4ecef20 100644 --- a/internal/store/situation_notifications.go +++ b/internal/store/situation_notifications.go @@ -41,19 +41,39 @@ var ( ErrNotificationIntentSuperseded = situation.ErrNotificationIntentSuperseded ) -// notificationClaimOrder is the exact claim ordering the plan names: gap -// generation first (the installation recovery notice precedes every -// Situation's backlog), then Situation, then — within a Situation — the -// coalescible root projection ahead of the immutable journal, then +// ErrNewerRootProjectionPending means a root projection could not be +// returned to pending because a NEWER one already holds its Situation's +// single pending-root slot (migration 0018's +// notification_intents_root_sync_pending_idx). The newer projection renders +// the same current state, so the older one has nothing left to say — this +// is a refusal, never a constraint violation. +var ErrNewerRootProjectionPending = errors.New("store: a newer root projection is already pending") + +// The claim ordering the plan names: gap generation first (the installation +// recovery notice precedes every Situation's backlog), then Situation, then // Transition sequence, then creation identity. // -// notificationClassRank is that within-Situation rank as a SQL expression: -// root_sync (0) before thread_append (1) before broadcast_handoff (2), so a -// root edit for a handoff always delivers before its broadcast reply -// (spec.md "Local idempotency, external delivery, and ordering", rule 3). +// Effect class enters that order in exactly ONE place — the coalescible root +// projection sorts ahead of every reply (notificationRootFirst), which is the +// only class ordering the spec requires ("a root edit for a handoff delivers +// before its broadcast reply"; the root's own rank alone achieves it). +// Class must NOT outrank Transition sequence: doing so put every quiet +// thread_append ahead of every broadcast_handoff regardless of sequence, so +// an older poke could deliver after newer entries, against spec.md's +// "immutable journal replies deliver in Transition-sequence order". +// notificationClassRank therefore survives only as an intra-sequence +// tiebreak — the one case it decides is a floor-withheld poke's Transition, +// which carries both a broadcast_handoff and a quiet thread_append at the +// same sequence. const ( + notificationRootFirst = `(ni.effect_class <> 'root_sync')` notificationClassRank = `CASE ni.effect_class WHEN 'root_sync' THEN 0 WHEN 'thread_append' THEN 1 ELSE 2 END` - notificationClaimOrder = `ORDER BY (gap_generation IS NULL) ASC, gap_generation ASC, situation_id ASC, class_rank ASC, transition_sequence ASC, id ASC` + notificationQueueOrder = `root_first ASC, transition_sequence ASC, class_rank ASC, id ASC` + notificationClaimOrder = `ORDER BY (gap_generation IS NULL) ASC, gap_generation ASC, situation_id ASC, ` + notificationQueueOrder + // notificationReloadOrder is notificationClaimOrder expressed directly + // against the table (alias ni), for the post-claim reload. + notificationReloadOrder = `ORDER BY (ni.gap_generation IS NULL) ASC, ni.gap_generation ASC, ni.situation_id ASC, ` + + notificationRootFirst + ` ASC, ni.transition_sequence ASC, ` + notificationClassRank + ` ASC, ni.id ASC` ) // validateNotificationClaim rejects a claim that cannot fence anything. @@ -101,7 +121,8 @@ func validateNotificationErrorClass(class string) error { // dead-lettering them; and // - the HEAD of its Situation's queue. Exactly one intent per Situation // is claimable at a time, ranked root projection first and then by -// Transition sequence, so a later immutable entry can never pass an +// Transition SEQUENCE (effect class decides only ties within one +// sequence), so a later immutable entry can never pass an // earlier pending one — including one merely waiting out a retry delay. // // A gap generation gates the whole claim: while one is open nothing is @@ -244,13 +265,14 @@ func dueNotificationIntentIDsTx(ctx context.Context, tx *sql.Tx, gate deliveryGa ni.gap_generation AS gap_generation, ni.situation_id AS situation_id, ni.transition_sequence AS transition_sequence, + `+notificationRootFirst+` AS root_first, `+notificationClassRank+` AS class_rank, (ni.claim_owner IS NULL OR ni.lease_expires_at <= ?) AS unleased, (ni.retry_at IS NULL OR ni.retry_at <= ?) AS due, (ni.requires_root = 0 OR (s.slack_channel IS NOT NULL AND s.slack_root_ts IS NOT NULL)) AS root_ready, ROW_NUMBER() OVER ( PARTITION BY ni.situation_id - ORDER BY `+notificationClassRank+` ASC, ni.transition_sequence ASC, ni.id ASC + ORDER BY `+notificationRootFirst+` ASC, ni.transition_sequence ASC, `+notificationClassRank+` ASC, ni.id ASC ) AS rn FROM notification_intents ni LEFT JOIN situations s ON s.id = ni.situation_id @@ -279,11 +301,10 @@ func loadClaimedNotificationIntentsTx(ctx context.Context, tx *sql.Tx, ids []str placeholders, args := inPlaceholders(ids) args = append(args, owner) query := ` - SELECT ` + qualifyColumns(notificationIntentColumns, "ni") + `, - ` + notificationClassRank + ` AS class_rank + SELECT ` + qualifyColumns(notificationIntentColumns, "ni") + ` FROM notification_intents ni WHERE ni.id IN (` + placeholders + `) AND ni.claim_owner = ? - ` + notificationClaimOrder // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + ` + notificationReloadOrder // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return nil, fmt.Errorf("store: read claimed notification intents: %w", err) @@ -292,8 +313,7 @@ func loadClaimedNotificationIntentsTx(ctx context.Context, tx *sql.Tx, ids []str out := make([]situation.NotificationClaim, 0, len(ids)) for rows.Next() { - var classRank int - intent, err := scanNotificationIntent(suffixedScanner{rows: rows, suffix: []any{&classRank}}) + intent, err := scanNotificationIntent(rows) if err != nil { return nil, fmt.Errorf("store: scan claimed notification intent: %w", err) } @@ -305,21 +325,6 @@ func loadClaimedNotificationIntentsTx(ctx context.Context, tx *sql.Tx, ids []str return out, nil } -// suffixedScanner lets scanNotificationIntent consume a row carrying extra -// TRAILING columns (the ORDER BY's own class_rank), mirroring -// prefixedScanner one direction over. -type suffixedScanner struct { - rows *sql.Rows - suffix []any -} - -func (p suffixedScanner) Scan(dest ...any) error { - all := make([]any, 0, len(dest)+len(p.suffix)) - all = append(all, dest...) - all = append(all, p.suffix...) - return p.rows.Scan(all...) -} - // inPlaceholders builds a "?,?,..." run of len(values) and the matching // bound argument slice. func inPlaceholders(values []string) (string, []any) { @@ -533,27 +538,76 @@ func (s *Store) RecoverExpiredNotificationClaims(ctx context.Context, now time.T // intent to pending, due now, with its attempt count preserved. It is the // only way out of `failed`, and the way a failed root releases the // dependent history waiting behind it. +// +// A failed ROOT projection shares reactivation's uniqueness hazard: its +// Situation may have acquired a newer pending root_sync while this one sat +// failed. When the redriven projection is the newer of the two, the pending +// one is coalesced into it exactly as a newer commit would; when it is the +// OLDER, the redrive is refused with ErrNewerRootProjectionPending — the +// newer projection already renders the same current state — rather than +// aborting on migration 0018's index. func (s *Store) RedriveFailedNotificationIntent(ctx context.Context, intentID string, now time.Time) error { if strings.TrimSpace(intentID) == "" { return errors.New("store: notification redrive requires an intent id") } - res, err := s.db.ExecContext(ctx, ` - UPDATE notification_intents - SET status = 'pending', retry_at = ?, claim_owner = NULL, lease_expires_at = NULL - WHERE id = ? AND status = 'failed'`, canonicalTime(now.UTC()), intentID) + nowStr := canonicalTime(now.UTC()) + + tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("store: redrive failed notification intent: %w", err) + return fmt.Errorf("store: begin redrive failed notification intent: %w", err) + } + defer func() { _ = tx.Rollback() }() + + var effectClass, createdAt string + var situationID sql.NullString + err = tx.QueryRowContext(ctx, + `SELECT effect_class, situation_id, created_at FROM notification_intents WHERE id = ? AND status = 'failed'`, + intentID).Scan(&effectClass, &situationID, &createdAt) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("store: notification intent %s is not in failed status: %w", intentID, ErrNotFound) } - n, err := res.RowsAffected() if err != nil { - return fmt.Errorf("store: count redriven notification intent: %w", err) + return fmt.Errorf("store: read failed notification intent: %w", err) } - if n != 1 { - return fmt.Errorf("store: notification intent %s is not in failed status: %w", intentID, ErrNotFound) + + if situationmodel.EffectClass(effectClass) == situationmodel.EffectRootSync && situationID.Valid { + if err := clearPendingRootForRedriveTx(ctx, tx, situationID.String, intentID, createdAt); err != nil { + return err + } + } + if err := setNotificationIntentPendingTx(ctx, tx, intentID, "failed", nowStr); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("store: commit redrive failed notification intent: %w", err) } return nil } +// clearPendingRootForRedriveTx frees the Situation's single pending-root slot +// for the projection being redriven, or refuses when a newer projection +// already holds it. +func clearPendingRootForRedriveTx(ctx context.Context, tx *sql.Tx, situationID, intentID, createdAt string) error { + var pendingID, pendingCreatedAt string + err := tx.QueryRowContext(ctx, ` + SELECT id, created_at FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, situationID). + Scan(&pendingID, &pendingCreatedAt) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("store: read pending root projection for redrive: %w", err) + } + // created_at is canonical RFC3339Nano UTC, so string order is time + // order; the id breaks a same-instant tie the same way the claim + // ordering does. + if pendingCreatedAt > createdAt || (pendingCreatedAt == createdAt && pendingID > intentID) { + return ErrNewerRootProjectionPending + } + return supersedePendingRootSyncTx(ctx, tx, situationID, intentID) +} + // GetSituationRootCoordinates reads situationID's durable Slack root // coordinates. ok is false when no root has been delivered yet — the // coordinates are nullable COLUMNS on an existing row, so their absence is diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index f5b42f0..3a3156f 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -30,6 +30,18 @@ func snCommit(t *testing.T, st *Store, sitID string, prior *situation.Controller contract := shRunningTriageContract(now.Add(time.Minute)) if prior != nil { contract = shOperatorContract(now.Add(time.Minute)) + } + return snCommitWith(t, st, sitID, prior, contract, now) +} + +// snCommitWith is snCommit with an explicit Operator contract, so a test can +// choose whether a cycle qualifies as a main-channel poke (an +// OperatorActionRequired appearing for the first time is PokeOperatorHandoff; +// handing the next move back to AlertINT is material but PokeNone). +func snCommitWith(t *testing.T, st *Store, sitID string, prior *situation.ControllerCommit, + contract situationmodel.ActionContract, now time.Time) situation.ControllerCommit { + t.Helper() + if prior != nil { shMakeDue(t, st, sitID, now.Add(-time.Minute)) } claim := claimSituation(t, st, sitID, "controller-a", now) @@ -725,3 +737,209 @@ func TestNotificationClaimNeverClaimsAFloorWithheldEffect(t *testing.T) { t.Fatalf("withheld intent = %+v, want an untouched durable decision with no attempts", got) } } + +// TestNotificationClaimOrdersJournalBySequenceAcrossEffectClasses is the +// regression for a claim order that ranked effect class ABOVE Transition +// sequence: every thread_append then sorted ahead of every +// broadcast_handoff, so an older poke could be delivered after newer quiet +// entries. spec.md's ordering is "immutable journal replies deliver in +// Transition-sequence order" — the only class ordering it requires is that +// the root projection precede its replies. +// +// The fixture is deliberately the one shape the earlier ordering tests did +// not have: a broadcast_handoff at a LOWER sequence than a pending +// thread_append. +func TestNotificationClaimOrdersJournalBySequenceAcrossEffectClasses(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-claim-class-order", now) + // Cycle 2 hands the next move to an operator: a poke, so its journal + // entry is a broadcast_handoff at sequence 2. + second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) + // Cycle 3 takes the next move back: material, but PokeNone, so its + // journal entry is a quiet thread_append at sequence 3. + _ = snCommitWith(t, st, sitID, &second, shRunningTriageContract(now.Add(3*time.Minute)), now.Add(2*time.Minute)) + + root := snClaimOne(t, st, now.Add(4*time.Minute)) + if root.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("head = %q, want the root projection first", root.Intent.EffectClass) + } + snDeliver(t, st, root, "100.1", now.Add(4*time.Minute)) + + type entry struct { + sequence int + class string + } + got := []entry{} + for round := 0; round < 5; round++ { + at := now.Add(time.Duration(5+round) * time.Minute) + claims, err := st.ClaimNotificationIntents(context.Background(), snOwner, at, 5*time.Minute, 25) + if err != nil { + t.Fatalf("claim round %d: %v", round, err) + } + if len(claims) == 0 { + break + } + c := claims[0] + got = append(got, entry{sequence: *c.Intent.TransitionSequence, class: string(c.Intent.EffectClass)}) + snDeliver(t, st, c, "10"+string(rune('0'+round))+".2", at) + } + want := []entry{ + {sequence: 1, class: "thread_append"}, + {sequence: 2, class: "broadcast_handoff"}, + {sequence: 3, class: "thread_append"}, + } + if len(got) != len(want) { + t.Fatalf("delivered %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("delivered %v, want %v — the immutable journal must drain in Transition-sequence order, "+ + "never with every quiet entry ahead of an older broadcast", got, want) + } + } + _ = sitID +} + +// TestNotificationAckReactivationKeepsOnePendingRootProjection is the +// regression for a reactivation that returned EVERY blocked intent to +// pending in one statement: when a Situation had both a blocked root_sync +// and a newer pending one, that violated migration 0018's +// notification_intents_root_sync_pending_idx, rolled the whole transaction +// back, and left the configuration generation unadvanced — so nothing +// reactivated for any Situation, permanently. +// +// Case A: the newest projection is already pending. It renders the current +// state, so the older blocked one is a resolved outcome (migration 0018's +// own words) and stays blocked; reactivation must still succeed for +// everything else. +func TestNotificationAckReactivationKeepsOnePendingRootProjection(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-ack-reactivate-newer", now) + firstRoot := shIntentOfClass(t, first.History.Intents, situationmodel.EffectRootSync) + + blocked := snClaimOne(t, st, now) + if blocked.Intent.ID != firstRoot.ID { + t.Fatalf("claimed %s, want the first root %s", blocked.Intent.ID, firstRoot.ID) + } + if err := st.BlockNotificationConfiguration(ctx, blocked, "channel_not_found", now); err != nil { + t.Fatalf("BlockNotificationConfiguration: %v", err) + } + // A later material commit inserts a NEW root projection. Supersession + // only ever retires a pending one, so the blocked one survives beside it. + second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) + secondRoot := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) + + n, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(2*time.Minute)) + if err != nil { + t.Fatalf("ReactivateConfigurationBlocked with a newer pending root: %v", err) + } + if n != 0 { + t.Fatalf("reactivated %d root projections, want 0 — the newer pending one already renders current state", n) + } + if got := snIntent(t, st, secondRoot.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("newer root status = %q, want pending", got.Status) + } + if got := snIntent(t, st, firstRoot.ID); got.Status == situationmodel.IntentPending { + t.Fatal("the superseded-by-newer blocked root must not be returned to pending") + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, + sitID); n != 1 { + t.Fatalf("pending root projections = %d, want exactly 1", n) + } + // The configuration generation really advanced, so the whole + // reactivation transaction committed. + if state := snState(t, st); state.ConfigurationGeneration != 1 { + t.Fatalf("configuration generation = %d, want the advanced 1", state.ConfigurationGeneration) + } +} + +// Case B: every live root projection for the Situation is blocked. The +// newest is the one corrected configuration should deliver; the older ones +// are coalesced into it, exactly as a newer projection always coalesces an +// older one. +func TestNotificationAckReactivationCoalescesOlderBlockedRootProjections(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-ack-reactivate-both", now) + firstRoot := shIntentOfClass(t, first.History.Intents, situationmodel.EffectRootSync) + + blocked := snClaimOne(t, st, now) + if err := st.BlockNotificationConfiguration(ctx, blocked, "channel_not_found", now); err != nil { + t.Fatalf("BlockNotificationConfiguration: %v", err) + } + second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) + secondRoot := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) + blockedAgain := snClaimOne(t, st, now.Add(2*time.Minute)) + if blockedAgain.Intent.ID != secondRoot.ID { + t.Fatalf("claimed %s, want the second root %s", blockedAgain.Intent.ID, secondRoot.ID) + } + if err := st.BlockNotificationConfiguration(ctx, blockedAgain, "channel_not_found", now.Add(2*time.Minute)); err != nil { + t.Fatalf("BlockNotificationConfiguration: %v", err) + } + + n, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(3*time.Minute)) + if err != nil { + t.Fatalf("ReactivateConfigurationBlocked with two blocked roots: %v", err) + } + if n < 1 { + t.Fatalf("reactivated %d intents, want at least the newest root projection", n) + } + if got := snIntent(t, st, secondRoot.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("newest root status = %q, want pending", got.Status) + } + older := snIntent(t, st, firstRoot.ID) + if older.Status != situationmodel.IntentSuperseded { + t.Fatalf("older root status = %q, want superseded by the newest projection", older.Status) + } + if older.ReplacementIntentID == nil || *older.ReplacementIntentID != secondRoot.ID { + t.Fatalf("older root replacement = %v, want %q", older.ReplacementIntentID, secondRoot.ID) + } + if older.SupersessionReason == nil || *older.SupersessionReason == "" { + t.Fatal("a superseded root projection must record why") + } + if n := shCountRows(t, st, + `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, + sitID); n != 1 { + t.Fatalf("pending root projections = %d, want exactly 1", n) + } + // The reactivated root really is claimable and deliverable. + claim := snClaimOne(t, st, now.Add(4*time.Minute)) + if claim.Intent.ID != secondRoot.ID { + t.Fatalf("claimed %s after reactivation, want the reactivated root %s", claim.Intent.ID, secondRoot.ID) + } +} + +// TestNotificationAckRedriveRefusesBehindANewerRootProjection proves the +// same uniqueness hazard cannot reach an operator redrive either: redriving +// an older failed root behind a newer pending one is refused with a typed +// error, not a raw constraint violation. +func TestNotificationAckRedriveRefusesBehindANewerRootProjection(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-ack-redrive-newer", now) + firstRoot := shIntentOfClass(t, first.History.Intents, situationmodel.EffectRootSync) + + failed := snClaimOne(t, st, now) + if err := st.FailNotificationIntent(ctx, failed, "invalid_payload", now); err != nil { + t.Fatalf("FailNotificationIntent: %v", err) + } + second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) + secondRoot := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) + + err := st.RedriveFailedNotificationIntent(ctx, firstRoot.ID, now.Add(2*time.Minute)) + if !errors.Is(err, ErrNewerRootProjectionPending) { + t.Fatalf("redrive behind a newer pending root = %v, want ErrNewerRootProjectionPending", err) + } + if got := snIntent(t, st, firstRoot.ID); got.Status != situationmodel.IntentFailed { + t.Fatalf("refused redrive changed the failed root to %q", got.Status) + } + if got := snIntent(t, st, secondRoot.ID); got.Status != situationmodel.IntentPending { + t.Fatalf("newer root status = %q, want an untouched pending", got.Status) + } +} From 37d45688846c738da9866faa3da749a94be72d9c Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 07:06:45 +0300 Subject: [PATCH 14/31] refactor(notify): cut over to Situation Slack ownership Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- cmd/alertint/main.go | 81 +++--- cmd/alertint/main_test.go | 77 ++++++ internal/correlator/attach.go | 11 + .../correlator/attach_integration_test.go | 53 ++++ internal/correlator/correlator.go | 38 ++- internal/notify/resolution/resolution.go | 84 ------ internal/notify/resolution/resolution_test.go | 77 ------ internal/store/annotations.go | 72 ++++- internal/store/annotations_test.go | 250 ++++++++++++++++++ internal/store/verdicts.go | 10 + internal/store/verdicts_test.go | 206 +++++++++++++++ skills/acutetriage/capture.go | 51 ++-- skills/acutetriage/capture_test.go | 180 +++++++++++-- 13 files changed, 934 insertions(+), 256 deletions(-) delete mode 100644 internal/notify/resolution/resolution.go delete mode 100644 internal/notify/resolution/resolution_test.go diff --git a/cmd/alertint/main.go b/cmd/alertint/main.go index 25bbc1c..83ae119 100644 --- a/cmd/alertint/main.go +++ b/cmd/alertint/main.go @@ -51,7 +51,6 @@ import ( "github.com/alertint/alertint-agent/internal/logs/loki" internalmcp "github.com/alertint/alertint-agent/internal/mcp" "github.com/alertint/alertint-agent/internal/notify" - notifyresolution "github.com/alertint/alertint-agent/internal/notify/resolution" notifyslack "github.com/alertint/alertint-agent/internal/notify/slack" notifystdout "github.com/alertint/alertint-agent/internal/notify/stdout" promclient "github.com/alertint/alertint-agent/internal/prometheus" @@ -389,17 +388,29 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { } cor := correlator.New(corCfg, st, productionIncidentSink(), logger) - // SetTriageFailureNotifier is safe to wire here, before reconstruction: - // it is not reachable from ApplyDelivery's durable-dispatch path (the - // triage-exhausted notifier fires only from the correlator's own - // internal ticker loop, which isn't running during reconstruction). + // Task 8: production wires NO notifier onto the Correlator at all any + // more — not SetTriageFailureNotifier here, and not + // SetResolutionNotifier/SetOccurrenceNotifier inside startCorrelator + // below either. The Situation notification worker (Task 6/7) is now the + // sole production Slack writer; the Correlator's own + // ResolutionNotifier/OccurrenceNotifier/TriageFailureNotifier setters + // stay real Go shape (this package's own tests, and + // internal/correlator's, keep exercising them with fakes) but are never + // handed a live instance here. Each domain outcome's durable Situation + // input (incident_resolved, membership_changed for an occurrence + // attach, triage_exhausted) is written directly by domain logic inside + // the relevant atomic store commit — see correlator.go's own doc + // comments on the three interfaces — so leaving all three unwired loses + // no durable history, only the retired Incident-card Slack/stdout + // fan-out. + // // The Correlator has no analyzer/LLM seam at all — no IncidentSink // beyond the no-op one and no re-judgment runner (Plan 2 Task 7) — // see TestProductionCorrelatorHasNoAcuteTriageDispatchDependency. - // SetAuditor, SetResolutionNotifier, and SetOccurrenceNotifier are NOT - // wired here — all three ARE reachable from ApplyDelivery — see - // startCorrelator below. - cor.SetTriageFailureNotifier(notifier) + // SetAuditor IS still wired — inside startCorrelator below, alongside + // this comment's former SetResolutionNotifier/SetOccurrenceNotifier + // neighbors — because it alone remains reachable from ApplyDelivery's + // durable-dispatch path. // stopCorrelator is called exactly once, however runServe exits: inline, // in the right relative position, by foundationStopSequence on the @@ -466,25 +477,26 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { backfillAndRecoverControllerWork: func(ctx context.Context) error { return runControllerRecovery(ctx, crt, logger) }, - // SetAuditor/SetResolutionNotifier/SetOccurrenceNotifier are wired - // here — between reconstruct and cor.Start, never before — because - // all three are synchronously reachable from ApplyDelivery's - // durable-dispatch path (a queued resolved delivery whose commit - // settles an Incident calls the resolution notifier; a queued firing - // delivery that collapses into a recurrence occurrence calls the - // occurrence notifier and appends the occurrence_attached audit - // event; a queued retry attach appends triage_member_attached). - // Wiring them before reconstruction would let a plain - // crash-and-restart with ordinary queued webhook traffic post to - // Slack — or append audit rows — from reconstruction: exactly the - // outward effects the spec's "reconstruction invokes no notifier, - // audit callback, ..." acceptance forbids. The Setters' own doc - // comments require only "after New, before Start", so this ordering - // is legal; the Correlator's loop itself isn't running yet either. + // SetAuditor is wired here — between reconstruct and cor.Start, never + // before — because it is synchronously reachable from ApplyDelivery's + // durable-dispatch path (a queued retry attach appends + // triage_member_attached; a queued firing delivery that collapses + // into a recurrence occurrence appends occurrence_attached). + // Wiring it before reconstruction would let a plain crash-and-restart + // with ordinary queued webhook traffic append audit rows from + // reconstruction: exactly the outward effect the spec's + // "reconstruction invokes no notifier, audit callback, ..." + // acceptance forbids. The Setter's own doc comment requires only + // "after New, before Start", so this ordering is legal; the + // Correlator's loop itself isn't running yet either. + // + // Task 8: SetResolutionNotifier/SetOccurrenceNotifier are NOT called + // here (or anywhere in production) any more — see the comment above + // cor's construction. Their durable Situation inputs are written + // directly by ApplyCorrelatedDelivery, in the same atomic commit + // this dispatch path already runs, independent of any notifier. startCorrelator: func(ctx context.Context) error { cor.SetAuditor(auditor) - cor.SetResolutionNotifier(notifyresolution.New(notifier, st)) - cor.SetOccurrenceNotifier(notifier) return cor.Start(ctx) }, startWorkers: rt.Start, @@ -1110,11 +1122,14 @@ func buildHealthChecks(cfg *config.Config, prom *promclient.Client, logSrc logs. // - stdout: always an active sink when notify.stdout is set, so a send is // confirmed (notified · stdout=ok) at INFO. Its verbose full JSON line is // written only at debug level (consistently, in every format). -// - slack: when enabled and a bot token resolves. // -// buildNotifier also returns the llmhealth.Publisher for the installation's -// one system-message surface: the same Slack *Notifier when Slack is wired, -// else nil (LLM dependency health then lives in state/audit/logs only). +// Task 8: Slack is never registered into this Incident fan-out any more — +// the Situation notification worker (Task 6/7) is the sole production Slack +// writer for anything Incident-shaped now (findings, resolutions, occurrence +// attaches, annotations/Captured verdicts). buildNotifier still constructs +// the concrete Slack *Notifier and returns it as the llmhealth.Publisher +// below when Slack is enabled and its bot token resolves — that remaining +// production Slack use (ADR-0042/0046 System messages) is unaffected. func buildNotifier(cfg *config.Config, st *store.Store, auditor *audit.Auditor, logger *slog.Logger, debug bool) (*notify.Multi, llmhealth.Publisher) { var nn []notify.Notifier var sinks []string @@ -1126,9 +1141,11 @@ func buildNotifier(cfg *config.Config, st *store.Store, auditor *audit.Auditor, } if cfg.Notify.Slack.Enabled { if token, err := cfg.SlackBotToken(); err == nil && token != "" { + // Constructed for the System-message surface only + // (ADR-0042/0046, the llmhealth.Publisher return below) — never + // appended to nn: an Incident Slack card or thread reply is the + // Situation notification worker's job now, not this fan-out's. slackNotifier := notifyslack.New(token, cfg.Notify.Slack.Channel, cfg.Notify.Slack.MinSeverity, cfg.Notify.Slack.RecurrenceMode, st, auditor) - nn = append(nn, slackNotifier) - sinks = append(sinks, "slack") slackWired = true publisher = slackNotifier } diff --git a/cmd/alertint/main_test.go b/cmd/alertint/main_test.go index 878a896..8cfb67f 100644 --- a/cmd/alertint/main_test.go +++ b/cmd/alertint/main_test.go @@ -6,12 +6,14 @@ import ( "bytes" "go/parser" "go/token" + "log/slog" "os" "path/filepath" "reflect" "strings" "testing" + "github.com/alertint/alertint-agent/internal/config" "github.com/alertint/alertint-agent/internal/correlator" "github.com/alertint/alertint-agent/internal/situation" "github.com/alertint/alertint-agent/skills/acutetriage" @@ -109,6 +111,81 @@ var ( _ situation.MinimumMemberAlertsPolicy = (*acutetriage.Skill)(nil) ) +// ---------------------------------------------------------------------- +// Task 8 one-writer topology proofs: the Situation notification worker +// (Task 6/7) is the sole production Slack writer for anything Incident- +// shaped. internal/notify/slack/system.go (ADR-0042/0046 System messages) +// and llmhealth's own publisher stay the one other reachable Slack surface — +// unaffected by these checks. +// ---------------------------------------------------------------------- + +// TestMainAssembly_BuildNotifierNeverRegistersSlackForIncidentFanout proves +// buildNotifier's Incident notify.Multi registration never includes a +// Slack-backed sink, even when Slack is fully enabled and its bot token +// resolves — Task 8 removed exactly that one `nn = append(nn, slackNotifier)` +// line. The llmhealth.Publisher return (the same concrete Slack *Notifier, +// wired for ADR-0042/0046 System messages only) is untouched and must stay +// non-nil. +func TestMainAssembly_BuildNotifierNeverRegistersSlackForIncidentFanout(t *testing.T) { + cfg := config.Defaults() + cfg.Notify.Stdout = true + cfg.Notify.Slack.Enabled = true + cfg.Notify.Slack.BotTokenEnv = "ALERTINT_TEST_SLACK_OWNERSHIP_TOKEN" + cfg.Notify.Slack.Channel = "#alerts" + t.Setenv("ALERTINT_TEST_SLACK_OWNERSHIP_TOKEN", "xoxb-test") + + multi, pub := buildNotifier(&cfg, nil, nil, slog.Default(), false) + if multi == nil { + t.Fatal("buildNotifier returned a nil *notify.Multi") + } + if pub == nil { + t.Fatal("buildNotifier must still return a non-nil llmhealth.Publisher when Slack resolves (ADR-0042/0046 System messages) — Task 8 only removes Slack from the Incident fan-out, not the System-message surface") + } + + notifiers := reflect.ValueOf(*multi).FieldByName("notifiers") + if !notifiers.IsValid() { + t.Fatal("notify.Multi has no 'notifiers' field any more — update this structural check") + } + if notifiers.Len() == 0 { + t.Fatal("buildNotifier registered no sinks at all with stdout+slack both configured") + } + for i := 0; i < notifiers.Len(); i++ { + elemType := notifiers.Index(i).Elem().Type() + named := elemType + if named.Kind() == reflect.Pointer { + named = named.Elem() + } + if strings.Contains(named.PkgPath(), "/internal/notify/slack") { + t.Fatalf("buildNotifier registered a Slack-backed notifier (%s, package %s) into the Incident fan-out — Task 8 requires the Situation notification worker to be the sole Slack writer", elemType, named.PkgPath()) + } + } +} + +// TestMainAssembly_NeverWiresLegacyIncidentNotifiersIntoCorrelator is a +// source-text scan (not just "did it compile") proving cmd/alertint never +// calls SetResolutionNotifier, SetOccurrenceNotifier, or +// SetTriageFailureNotifier on the Correlator any more — the three legacy +// Incident-shaped notifier injections Task 8 removed. Their durable +// Situation inputs (incident_resolved, membership_changed for an occurrence +// attach, triage_exhausted) are written directly by domain logic inside the +// relevant atomic store commit, independent of any notifier — see +// correlator.go's ResolutionNotifier/OccurrenceNotifier/TriageFailureNotifier +// doc comments. +func TestMainAssembly_NeverWiresLegacyIncidentNotifiersIntoCorrelator(t *testing.T) { + src, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"SetResolutionNotifier(", "SetOccurrenceNotifier(", "SetTriageFailureNotifier("} { + if strings.Contains(string(src), forbidden) { + t.Errorf("main.go calls %s — production must leave the Correlator's legacy Incident notifier setters unwired", forbidden) + } + } + if strings.Contains(string(src), "notify/resolution") { + t.Error("main.go still references internal/notify/resolution — Task 8 deletes that adapter package") + } +} + func TestRun_VersionFlagPrintsVersionAndExitsCleanly(t *testing.T) { var stdout, stderr bytes.Buffer if err := run([]string{"--version"}, &stdout, &stderr); err != nil { diff --git a/internal/correlator/attach.go b/internal/correlator/attach.go index f22a796..3d2b6f1 100644 --- a/internal/correlator/attach.go +++ b/internal/correlator/attach.go @@ -331,6 +331,17 @@ func memberBaselines(members []store.Alert, incomingFP string) (maxSev int, maxS // and — for an escalation — leaves the trigger on the occurrence row. No // re-judgment runs here or anywhere else in this package: the Correlator // owns grouping and readiness only, never analyzer/LLM dispatch. +// +// This whole path (maybeAttachOccurrence/attachOccurrence, reached only via +// handleAlert) has been unreachable from any production Receiver since Task +// 4 — production correlates through the durable ApplyDelivery/ +// applyRecurrenceDeliveryPlan path in correlator.go instead, which writes +// the equivalent "membership_changed" Situation input directly inside +// ApplyCorrelatedDelivery's atomic commit. c.occNotifier here is exercised +// only by this file's own legacy in-memory fixtures (attach_integration_test.go); +// Task 8 does not need to touch it to close off Slack reachability — that is +// already true — but see correlator.go's OccurrenceNotifier doc for +// production's own (equally nil-by-default) wiring since Task 8. func (c *Correlator) attachOccurrence(ctx context.Context, a store.Alert, inc store.Incident, gk string, decision attachDecision, delta recurrenceDelta) error { occ := store.Occurrence{ IncidentID: inc.ID, diff --git a/internal/correlator/attach_integration_test.go b/internal/correlator/attach_integration_test.go index 2de6f26..1748f4a 100644 --- a/internal/correlator/attach_integration_test.go +++ b/internal/correlator/attach_integration_test.go @@ -570,3 +570,56 @@ func TestMaybeAttach_EventCarriesCadenceDelta(t *testing.T) { t.Errorf("cadence delta = {new:%s median:%s}, want new*8 < median with both > 0", ev.NewInterval, ev.PriorMedian) } } + +// TestApplyDelivery_ResolutionAndOccurrenceWorkWithNoNotifiersWired pins +// Task 8's new production default: cmd/alertint no longer calls +// SetResolutionNotifier/SetOccurrenceNotifier/SetTriageFailureNotifier at +// all — the Situation notification worker (Task 6/7) is now the sole +// production Slack writer, and a legacy Incident-shaped notifier is never +// constructed there any more. ResolutionNotifier/OccurrenceNotifier/ +// TriageFailureNotifier stay real Go interfaces the Correlator can still be +// wired with (this package's own tests keep doing exactly that with fakes, +// e.g. TestApplyDelivery_RecurrenceCollapseAttachesOccurrenceAndNotifies / +// TestApplyDelivery_ResolvedDeliveryResolvesIncidentAndNotifies in +// delivery_test.go); production simply leaves all three nil now. This test +// proves both durable ApplyDelivery outcomes — a resolved delivery flipping +// an Incident to "resolved" and a firing re-fire collapsing into an +// occurrence — commit their Incident/Occurrence mutation and Situation input +// correctly with every notifier left nil, since notification was always a +// best-effort side effect layered strictly AFTER the durable commit, never +// a dependency of it (correlator.go's applyResolvedDeliveryPlan/ +// applyRecurrenceDeliveryPlan call ApplyCorrelatedDelivery first and only +// then check "if c.resolutionNotifier != nil" / "if c.occNotifier != nil"). +func TestApplyDelivery_ResolutionAndOccurrenceWorkWithNoNotifiersWired(t *testing.T) { + t.Run("resolution", func(t *testing.T) { + st := openStore(t) + c := New(Config{}, st, NopIncidentSink{}, nil) // no Set*Notifier call at all + now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC) + member := firingAlert("fp-only", "DiskFull", "warning", now.Add(-time.Hour), false) + seedJudged(t, st, "inc_1", "ready", now.Add(-time.Hour), now.Add(-time.Hour), member) + + claim := claimOneDelivery(t, st, deliveryInputFor("d1", "fp-only", gkAPI, "resolved", now), now) + if err := c.ApplyDelivery(context.Background(), claim); err != nil { + t.Fatalf("resolved delivery with no notifier wired: %v", err) + } + inc, err := st.GetIncidentByID(context.Background(), "inc_1") + if err != nil || inc.Status != "resolved" { + t.Fatalf("incident must still resolve with no notifier wired: %+v, %v", inc, err) + } + }) + t.Run("occurrence", func(t *testing.T) { + st := openStore(t) + c := New(Config{}, st, NopIncidentSink{}, nil) // no Set*Notifier call at all + now := time.Date(2026, 9, 6, 3, 0, 0, 0, time.UTC) + member := firingAlert("fp-orig", "DiskFull", "warning", now.Add(-5*time.Minute), false) + seedJudged(t, st, "inc_1", "analyzed", now.Add(-5*time.Minute), now.Add(-10*time.Minute), member) + + claim := claimOneDelivery(t, st, deliveryInputFor("d1", "fp-new", gkAPI, "firing", now), now) + if err := c.ApplyDelivery(context.Background(), claim); err != nil { + t.Fatalf("recurrence collapse with no notifier wired: %v", err) + } + if occCount(t, st, "inc_1") != 1 { + t.Fatalf("occurrence must still attach with no notifier wired: %d", occCount(t, st, "inc_1")) + } + }) +} diff --git a/internal/correlator/correlator.go b/internal/correlator/correlator.go index c004501..65da353 100644 --- a/internal/correlator/correlator.go +++ b/internal/correlator/correlator.go @@ -55,8 +55,17 @@ type IncidentSink interface { OnIncidentReady(ctx context.Context, inc store.Incident) error } -// ResolutionNotifier receives notifications when an incident becomes fully resolved -// (all alerts have status="resolved"). +// ResolutionNotifier receives notifications when an incident becomes fully +// resolved (all alerts have status="resolved"). Task 8: cmd/alertint no +// longer calls SetResolutionNotifier with a Slack-backed instance in +// production — the Situation notification worker (Task 6/7) is now the sole +// production Slack writer, and the durable "incident_resolved" Situation +// input is written directly by ApplyCorrelatedDelivery/ +// correlatedSituationInputKindTx (internal/store/deliveries.go) inside the +// same atomic commit, strictly before this notifier (if any) is even +// considered — so leaving it nil costs nothing durable. The interface stays +// a real Go shape the Correlator can still be wired with (delivery_test.go's +// own fakes keep doing exactly that). type ResolutionNotifier interface { OnIncidentResolved(ctx context.Context, inc store.Incident) error } @@ -67,17 +76,32 @@ type NopIncidentSink struct{} func (NopIncidentSink) OnIncidentReady(_ context.Context, _ store.Incident) error { return nil } -// OccurrenceNotifier receives a deterministic, zero-LLM notification each time a -// re-fire attaches as an occurrence (recurrence collapse). The stdout notifier -// emits one line; the Slack notifier edits the card and/or posts the "why" as a -// thread reply. nil means no occurrence notifications. +// OccurrenceNotifier receives a deterministic, zero-LLM notification each +// time a re-fire attaches as an occurrence (recurrence collapse). nil means +// no occurrence notifications — which is production's own default as of +// Task 8: cmd/alertint no longer calls SetOccurrenceNotifier with a +// Slack-backed instance (the Situation notification worker, Task 6/7, is now +// the sole production Slack writer), and the durable "membership_changed" +// Situation input for the attach is written directly by +// ApplyCorrelatedDelivery inside the same atomic commit, strictly before +// this notifier (if any) is even considered. type OccurrenceNotifier interface { OnOccurrenceAttached(ctx context.Context, ev notify.RecurrenceEvent) error } // TriageFailureNotifier receives one event when an incident's triage has // exhausted its retry schedule and the incident was marked "failed". nil -// disables it. +// disables it — production's own default, both before and after Task 8: +// the exhaustion notification path that actually reaches an operator is +// skills/acutetriage.Skill.OnTriageExhausted (wired as TriageWorker's +// ExhaustionNotifier in cmd/alertint/situation_controller.go), never this +// Correlator field — c.triageNotifier has had no reachable caller inside +// this package since Task 7 removed the Correlator's own dispatch/ +// exhaustion chain, and cmd/alertint stopped calling SetTriageFailureNotifier +// in Task 8 accordingly. The durable "triage_exhausted" Situation input is +// written directly by TriageWorker's own store commit +// (internal/store/triage_controller.go's ExhaustIncidentTriageAttempt), +// independent of any notifier. type TriageFailureNotifier interface { OnTriageExhausted(ctx context.Context, ev notify.TriageExhaustedEvent) error } diff --git a/internal/notify/resolution/resolution.go b/internal/notify/resolution/resolution.go deleted file mode 100644 index 259df92..0000000 --- a/internal/notify/resolution/resolution.go +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: FSL-1.1-ALv2 - -// Package resolution implements a resolution notifier that wraps existing -// notifiers to send incident resolved notifications. -package resolution - -import ( - "context" - "encoding/json" - "time" - - "github.com/alertint/alertint-agent/internal/correlator" - "github.com/alertint/alertint-agent/internal/notify" - "github.com/alertint/alertint-agent/internal/store" -) - -// Notifier wraps existing notifiers to send resolution notifications. -type Notifier struct { - inner notify.Notifier - st *store.Store -} - -// New creates a resolution notifier that wraps an existing notifier. st may -// be nil (tests); it is used to re-derive the Drill flag so a resolving -// drill's in-place card update keeps its DRILL banner (ADR-0013). -func New(inner notify.Notifier, st *store.Store) *Notifier { - return &Notifier{inner: inner, st: st} -} - -// OnIncidentResolved implements correlator.ResolutionNotifier. -// It sends a resolution notification using the wrapped notifier. -func (n *Notifier) OnIncidentResolved(ctx context.Context, inc store.Incident) error { - // Carry original LLM analysis into the resolved finding when available - // so notifiers (e.g. Slack) can preserve context in the updated message. - analysisName := inc.Summary - if analysisName == "" { - analysisName = "Incident Resolved" - } - overallIssue := inc.RootCause - if overallIssue == "" { - overallIssue = "All alerts have recovered. Incident is now resolved." - } - confidence := inc.Confidence - if confidence == 0 { - confidence = 1.0 - } - - drill := false - if n.st != nil { - // Best-effort: a lookup failure must not block the resolution - // notification; the card just loses the banner in that edge. - if flags, err := n.st.IncidentDrillFlags(ctx, []string{inc.ID}); err == nil { - drill = flags[inc.ID] - } - } - - f := notify.Finding{ - IncidentID: inc.ID, - GroupKey: inc.GroupKey, - AnalysisName: analysisName, - OverallIssue: overallIssue, - Severity: "low", - Confidence: confidence, - AlertCount: inc.AlertCount, - FirstAlertAt: inc.FirstAlertAt, - AnalyzedAt: time.Now().UTC(), - OutputJSON: json.RawMessage(`{"status":"resolved"}`), - Status: "resolved", - Drill: drill, - } - if n.st != nil { - // Best-effort recurrence summary for the resolved card ("resolved after - // recurring ×N"). A lookup miss/err just omits it. - if m, err := n.st.OccurrenceStatsByIncident(ctx, []string{inc.ID}); err == nil { - if s, ok := m[inc.ID]; ok && s.Episodes() > 1 { - f.Recurrence = ¬ify.Recurrence{Episodes: s.Episodes(), LastSeen: s.LastSeen} - } - } - } - return n.inner.Notify(ctx, f) -} - -// Ensure Notifier implements correlator.ResolutionNotifier. -var _ correlator.ResolutionNotifier = (*Notifier)(nil) diff --git a/internal/notify/resolution/resolution_test.go b/internal/notify/resolution/resolution_test.go deleted file mode 100644 index c081e6a..0000000 --- a/internal/notify/resolution/resolution_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: FSL-1.1-ALv2 - -package resolution - -import ( - "context" - "testing" - "time" - - "github.com/alertint/alertint-agent/internal/notify" - "github.com/alertint/alertint-agent/internal/store" -) - -type captureNotifier struct { - last *notify.Finding -} - -func (c *captureNotifier) Notify(_ context.Context, f notify.Finding) error { - c.last = &f - return nil -} - -func (c *captureNotifier) Name() string { return "capture" } - -// TestOnIncidentResolved_PreservesDrill: a resolving drill's finding keeps -// Drill=true so the in-place Slack card update keeps its banner (ADR-0013). -func TestOnIncidentResolved_PreservesDrill(t *testing.T) { - ctx := context.Background() - st, err := store.Open(ctx, ":memory:") - if err != nil { - t.Fatalf("open store: %v", err) - } - t.Cleanup(func() { _ = st.Close() }) - - now := time.Now().UTC() - cases := map[string]struct { - labels map[string]string - want bool - }{ - "drill": {labels: map[string]string{store.DrillMarkerLabel: store.DrillMarkerValue}, want: true}, - "real": {labels: map[string]string{"service": "checkout"}, want: false}, - } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - inc := store.Incident{ID: "inc-" + name, GroupKey: "g=" + name, FirstAlertAt: now, LastAlertAt: now, ReadyAt: now} - if err := st.InsertIncident(ctx, inc); err != nil { - t.Fatal(err) - } - a := store.Alert{ID: name + "-a", Fingerprint: name + "-fp", Status: "resolved", Labels: tc.labels, Annotations: map[string]string{}, StartsAt: now, ReceivedAt: now} - if _, err := st.UpsertAlertByFingerprint(ctx, a); err != nil { - t.Fatal(err) - } - if err := st.AddAlertToIncident(ctx, inc.ID, a.ID, now); err != nil { - t.Fatal(err) - } - - capture := &captureNotifier{} - if err := New(capture, st).OnIncidentResolved(ctx, inc); err != nil { - t.Fatalf("OnIncidentResolved: %v", err) - } - if capture.last == nil || capture.last.Drill != tc.want { - t.Fatalf("Drill = %+v, want %v", capture.last, tc.want) - } - }) - } - - t.Run("nil store degrades to false without panic", func(t *testing.T) { - capture := &captureNotifier{} - inc := store.Incident{ID: "inc-nil", GroupKey: "g", FirstAlertAt: now, LastAlertAt: now} - if err := New(capture, nil).OnIncidentResolved(ctx, inc); err != nil { - t.Fatalf("OnIncidentResolved: %v", err) - } - if capture.last == nil || capture.last.Drill { - t.Fatalf("Drill = %+v, want false", capture.last) - } - }) -} diff --git a/internal/store/annotations.go b/internal/store/annotations.go index 7b62a5a..15c5b23 100644 --- a/internal/store/annotations.go +++ b/internal/store/annotations.go @@ -45,7 +45,12 @@ func validateAnnotation(kind, note string) error { } // InsertIncidentAnnotation appends one annotation row. Returns ErrNotFound -// when the incident does not exist. +// when the incident does not exist. Task 8: in the SAME transaction, when +// the Incident currently belongs to a nonterminal Situation, this also +// enqueues exactly one operator_annotation_recorded situation_input_outbox +// row referencing the new annotation — see enqueueOperatorArtifactInputTx. +// With no owner at all (or an already-terminal one), only the annotation is +// persisted; it stays visible through Incident MCP/audit either way. func (s *Store) InsertIncidentAnnotation(ctx context.Context, incidentID, kind, note string) (*IncidentAnnotation, error) { if err := validateAnnotation(kind, note); err != nil { return nil, err @@ -59,6 +64,10 @@ func (s *Store) InsertIncidentAnnotation(ctx context.Context, incidentID, kind, if err != nil { return nil, err } + idempotencyKey := fmt.Sprintf("operator-annotation:%d", a.ID) + if err := enqueueOperatorArtifactInputTx(ctx, tx, incidentID, "operator_annotation_recorded", idempotencyKey, a.ID, nil, a.CreatedAt); err != nil { + return nil, err + } if err := tx.Commit(); err != nil { return nil, fmt.Errorf("store: commit annotation: %w", err) } @@ -90,6 +99,67 @@ func insertAnnotationTx(ctx context.Context, tx *sql.Tx, incidentID, kind, note return &IncidentAnnotation{ID: id, IncidentID: incidentID, Kind: kind, Note: note, CreatedAt: now}, nil } +// enqueueOperatorArtifactInputTx atomically enqueues one situation_input_outbox +// row for a durable operator artifact — an attributed annotation +// (kind="operator_annotation_recorded", annotationID set, verdictID nil) or +// a Captured verdict (kind="captured_verdict_recorded", verdictID set, +// annotationID nil) — that the caller already persisted earlier in this +// SAME transaction (insertAnnotationTx / PersistVerdictCapture's verdict +// insert). Shared by both InsertIncidentAnnotation and PersistVerdictCapture +// (verdicts.go). +// +// It enqueues ONLY when incidentID currently belongs to a Situation whose +// lifecycle is nonterminal right now: an Incident with no owning Situation +// at all keeps the artifact visible only through Incident MCP/audit (no +// outbox row at all — situation_incidents' link is permanent once made, so +// situationOwnerForIncidentTx alone cannot tell "never owned" from "owned by +// a since-terminalized Situation", hence the separate lifecycle check +// below), and an Incident whose owner has ALREADY reached a terminal +// lifecycle gets none either — enqueuing there would be pure waste, since +// ApplySituationInput's R2 owner-terminal handling could never journal it. +// R2 exists for the genuine RACE where the owner terminalizes strictly +// BETWEEN this enqueue and the input worker's later apply, which this +// write-time check neither needs to nor can prevent. +// +// idempotencyKey must be derived deterministically from the artifact's own +// row id (see callers) so a retried enqueue for the exact same annotation/ +// verdict never creates a second input: ON CONFLICT(idempotency_key) DO +// NOTHING mirrors insertTriageSituationInputTx's own idempotency convention +// (triage_controller.go) and ApplyCorrelatedDelivery's outbox insert +// (deliveries.go). +func enqueueOperatorArtifactInputTx(ctx context.Context, tx *sql.Tx, incidentID, kind, idempotencyKey string, annotationID, verdictID any, occurredAt time.Time) error { + ownerID, err := situationOwnerForIncidentTx(ctx, tx, incidentID) + if err != nil { + return err + } + if ownerID == "" { + return nil + } + lifecycle, _, err := situationLifecycleAndVersionTx(ctx, tx, ownerID) + if err != nil { + if errors.Is(err, ErrNotFound) { + return nil + } + return err + } + if lifecycle.Terminal() { + return nil + } + var groupKey string + if err := tx.QueryRowContext(ctx, `SELECT group_key FROM incidents WHERE id = ?`, incidentID).Scan(&groupKey); err != nil { + return fmt.Errorf("store: read incident group key for %s: %w", kind, err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO situation_input_outbox + (id, idempotency_key, incident_id, kind, group_key, occurred_at, status, annotation_id, verdict_id, journal_state) + VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, 'pending') + ON CONFLICT(idempotency_key) DO NOTHING`, + "situation-input:"+idempotencyKey, idempotencyKey, incidentID, kind, groupKey, canonicalTime(occurredAt), annotationID, verdictID); err != nil { + return fmt.Errorf("store: enqueue %s situation input: %w", kind, err) + } + return nil +} + // ListIncidentAnnotations returns every annotation of one incident, // newest-first. func (s *Store) ListIncidentAnnotations(ctx context.Context, incidentID string) ([]IncidentAnnotation, error) { diff --git a/internal/store/annotations_test.go b/internal/store/annotations_test.go index cf3813b..c7bb8f1 100644 --- a/internal/store/annotations_test.go +++ b/internal/store/annotations_test.go @@ -5,8 +5,10 @@ package store import ( "context" "errors" + "fmt" "strings" "testing" + "time" ) func TestInsertAndListIncidentAnnotations(t *testing.T) { @@ -111,3 +113,251 @@ func TestOperatorAnnotations_Unbounded(t *testing.T) { t.Fatalf("want 1 permanent annotation, got %d, %v", len(ops), err) } } + +// ---------------------------------------------------------------------- +// Task 8: atomic write-back — an attributed annotation (and, in +// verdicts_test.go, a Captured verdict) atomically enqueues exactly one +// durable Situation input alongside the annotation/verdict row itself, but +// ONLY when the Incident currently belongs to a nonterminal Situation. +// ---------------------------------------------------------------------- + +// situationOwnedIncident builds one real Incident and durably attaches it to +// a brand-new, active Situation the same way the durable pipeline does — +// InsertIncident, a queued "incident_created" situation_input_outbox row, +// ClaimSituationInputs, ApplySituationInput — never an INSERT INTO +// situations by hand (mirrors internal/mcp's seedSituationForMCP). Returns +// the Incident id. +func situationOwnedIncident(t *testing.T, st *Store, groupKey string) string { + t.Helper() + ctx := context.Background() + now := time.Now().UTC() + incidentID := fmt.Sprintf("inc-%s-%d", groupKey, now.UnixNano()) + if err := st.InsertIncident(ctx, Incident{ + ID: incidentID, GroupKey: groupKey, FirstAlertAt: now, LastAlertAt: now, ReadyAt: now.Add(time.Minute), + }); err != nil { + t.Fatalf("insert incident: %v", err) + } + if err := st.MarkIncidentReady(ctx, incidentID); err != nil { + t.Fatalf("mark incident ready: %v", err) + } + inputID := "input-" + incidentID + if _, err := st.db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox (id, idempotency_key, incident_id, kind, group_key, occurred_at, status) + VALUES (?, ?, ?, 'incident_created', ?, ?, 'pending')`, + inputID, "idem:"+inputID, incidentID, groupKey, canonicalTime(now)); err != nil { + t.Fatalf("seed situation input: %v", err) + } + claim := claimOneInput(t, st, "seed-"+incidentID, now) + if err := st.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("apply seed situation input: %v", err) + } + return incidentID +} + +// situationIDForIncident reads the Situation id an Incident is durably +// attached to (situation_incidents is append-only — the link never changes +// even after the owner terminalizes). +func situationIDForIncident(t *testing.T, st *Store, incidentID string) string { + t.Helper() + var id string + if err := st.db.QueryRowContext(context.Background(), + `SELECT situation_id FROM situation_incidents WHERE incident_id = ?`, incidentID).Scan(&id); err != nil { + t.Fatalf("situation id for incident %s: %v", incidentID, err) + } + return id +} + +// terminalizeSituation marks situationID terminal (closed_unknown) directly, +// mirroring situations_test.go's own R2 fixture pattern. +func terminalizeSituation(t *testing.T, st *Store, situationID string, at time.Time) { + t.Helper() + if _, err := st.db.ExecContext(context.Background(), ` + UPDATE situations SET lifecycle='closed_unknown', terminal_at=?, terminal_reason='resolution_missing', updated_at=? + WHERE id=?`, canonicalTime(at), canonicalTime(at), situationID); err != nil { + t.Fatalf("terminalize situation %s: %v", situationID, err) + } +} + +// countOutboxRows counts situation_input_outbox rows of kind for incidentID — +// the write-time enqueue's own visible effect. +func countOutboxRows(t *testing.T, st *Store, incidentID, kind string) int { + t.Helper() + var n int + if err := st.db.QueryRowContext(context.Background(), ` + SELECT COUNT(*) FROM situation_input_outbox WHERE incident_id = ? AND kind = ?`, incidentID, kind).Scan(&n); err != nil { + t.Fatalf("count outbox rows: %v", err) + } + return n +} + +// TestInsertIncidentAnnotation_EnqueuesOperatorAnnotationRecordedWhenOwnerActive +// proves Step 2's core contract: in the SAME transaction that persists the +// annotation, an active owning Situation gets exactly one +// operator_annotation_recorded situation_input_outbox row referencing the +// exact new annotation id, pending and ready for the input worker. +func TestInsertIncidentAnnotation_EnqueuesOperatorAnnotationRecordedWhenOwnerActive(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=annotate-active") + + a, err := s.InsertIncidentAnnotation(ctx, id, "observation", "operator note") + if err != nil { + t.Fatalf("insert: %v", err) + } + + if n := countOutboxRows(t, s, id, "operator_annotation_recorded"); n != 1 { + t.Fatalf("operator_annotation_recorded inputs = %d, want 1", n) + } + var annotationID int64 + var status, journalState, groupKey string + if err := s.db.QueryRowContext(ctx, ` + SELECT annotation_id, status, journal_state, group_key FROM situation_input_outbox + WHERE incident_id = ? AND kind = 'operator_annotation_recorded'`, id). + Scan(&annotationID, &status, &journalState, &groupKey); err != nil { + t.Fatal(err) + } + if annotationID != a.ID { + t.Fatalf("annotation_id = %d, want %d (the exact annotation this call persisted)", annotationID, a.ID) + } + if status != "pending" || journalState != "pending" { + t.Fatalf("status=%q journal_state=%q, want pending/pending", status, journalState) + } + if groupKey != "service=annotate-active" { + t.Fatalf("group_key = %q, want the incident's own", groupKey) + } +} + +// TestInsertIncidentAnnotation_NoEnqueueWithoutOwner proves the negative +// half of Step 2: an Incident with no owning Situation at all persists the +// annotation (still visible via ListIncidentAnnotations/Incident MCP) but +// enqueues no situation_input_outbox row whatsoever. +func TestInsertIncidentAnnotation_NoEnqueueWithoutOwner(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := readyIncident(t, s, "service=annotate-unowned") + + if _, err := s.InsertIncidentAnnotation(ctx, id, "observation", "operator note"); err != nil { + t.Fatalf("insert: %v", err) + } + anns, err := s.ListIncidentAnnotations(ctx, id) + if err != nil || len(anns) != 1 { + t.Fatalf("annotation must still be visible: anns=%+v err=%v", anns, err) + } + if n := countOutboxRows(t, s, id, "operator_annotation_recorded"); n != 0 { + t.Fatalf("operator_annotation_recorded inputs = %d, want 0 (no owner at all)", n) + } +} + +// TestInsertIncidentAnnotation_NoEnqueueForTerminalOwner proves Step 4's +// "never enqueue for a terminal owner at write time": an Incident whose +// owning Situation has ALREADY reached a terminal lifecycle before the +// annotation is even written gets no outbox row — this is distinct from the +// R2 RACE (owner terminalizes strictly between enqueue and apply), covered +// separately below. +func TestInsertIncidentAnnotation_NoEnqueueForTerminalOwner(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=annotate-terminal") + situationID := situationIDForIncident(t, s, id) + terminalizeSituation(t, s, situationID, time.Now().UTC().Add(time.Hour)) + + if _, err := s.InsertIncidentAnnotation(ctx, id, "observation", "operator note"); err != nil { + t.Fatalf("insert: %v", err) + } + anns, err := s.ListIncidentAnnotations(ctx, id) + if err != nil || len(anns) != 1 { + t.Fatalf("annotation must still be visible: anns=%+v err=%v", anns, err) + } + if n := countOutboxRows(t, s, id, "operator_annotation_recorded"); n != 0 { + t.Fatalf("operator_annotation_recorded inputs = %d, want 0 (owner already terminal at write time)", n) + } +} + +// TestInsertIncidentAnnotation_RetrySameAnnotationIDIsIdempotent proves the +// enqueue's own idempotency: replaying the write-back enqueue for the exact +// same already-persisted annotation id (the retry scenario Step 2 names) +// never creates a second situation_input_outbox row. +func TestInsertIncidentAnnotation_RetrySameAnnotationIDIsIdempotent(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=annotate-retry") + annID, err := insertAnnotationRow(ctx, s, id) + if err != nil { + t.Fatalf("seed annotation: %v", err) + } + idempotencyKey := fmt.Sprintf("operator-annotation:%d", annID) + + for i := 0; i < 2; i++ { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := enqueueOperatorArtifactInputTx(ctx, tx, id, "operator_annotation_recorded", idempotencyKey, annID, nil, time.Now().UTC()); err != nil { + t.Fatalf("enqueue attempt %d: %v", i, err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + if n := countOutboxRows(t, s, id, "operator_annotation_recorded"); n != 1 { + t.Fatalf("operator_annotation_recorded inputs after retry = %d, want 1 (idempotent replay)", n) + } +} + +// TestInsertIncidentAnnotation_R2RaceOwnerTerminalizesBeforeApply is Step 4's +// full round trip: enqueue against an ACTIVE owner (this task's write path), +// terminalize the Situation, then run the input worker (Task 2's +// ApplySituationInput, already implemented) — the row lands owner_terminal, +// no Transition is ever created for it, input_version stays unchanged, and +// the annotation remains visible via Incident MCP/audit throughout. +func TestInsertIncidentAnnotation_R2RaceOwnerTerminalizesBeforeApply(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=annotate-r2-race") + situationID := situationIDForIncident(t, s, id) + + a, err := s.InsertIncidentAnnotation(ctx, id, "observation", "operator note") + if err != nil { + t.Fatalf("insert: %v", err) + } + before := getSituationByID(t, s, situationID) + + terminalizeSituation(t, s, situationID, time.Now().UTC().Add(time.Hour)) + + claim := claimOneInput(t, s, "input-worker", time.Now().UTC().Add(2*time.Hour)) + if err := s.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("apply artifact input to now-terminal owner: %v", err) + } + + after := getSituationByID(t, s, situationID) + if after.InputVersion != before.InputVersion { + t.Fatalf("input_version changed: before %d, after %d, want unchanged", before.InputVersion, after.InputVersion) + } + + var journalState string + var annotationID int64 + if err := s.db.QueryRowContext(ctx, ` + SELECT journal_state, annotation_id FROM situation_input_outbox + WHERE incident_id = ? AND kind = 'operator_annotation_recorded'`, id).Scan(&journalState, &annotationID); err != nil { + t.Fatal(err) + } + if journalState != "owner_terminal" { + t.Fatalf("journal_state = %q, want owner_terminal", journalState) + } + if annotationID != a.ID { + t.Fatalf("annotation_id = %d, want %d", annotationID, a.ID) + } + + var transitions int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, situationID).Scan(&transitions); err != nil { + t.Fatal(err) + } + if transitions != 0 { + t.Fatalf("situation_transitions for %s = %d, want 0 (owner_terminal is never journaled)", situationID, transitions) + } + + anns, err := s.ListIncidentAnnotations(ctx, id) + if err != nil || len(anns) != 1 { + t.Fatalf("annotation must remain visible via Incident MCP/audit: anns=%+v err=%v", anns, err) + } +} diff --git a/internal/store/verdicts.go b/internal/store/verdicts.go index aa8a337..e57d71c 100644 --- a/internal/store/verdicts.go +++ b/internal/store/verdicts.go @@ -112,6 +112,16 @@ func (s *Store) PersistVerdictCapture(ctx context.Context, c VerdictCapture) (*I return nil, nil, fmt.Errorf("store: verdict demotion: %w", err) } } + // Task 8: enqueue exactly one captured_verdict_recorded situation input — + // never operator_annotation_recorded, even though the verdict insert + // above (via insertAnnotationTx) also writes a matching + // incident_annotations row — under the same active/no-owner/terminal- + // owner rules InsertIncidentAnnotation uses (enqueueOperatorArtifactInputTx, + // annotations.go). + idempotencyKey := fmt.Sprintf("captured-verdict:%d", id) + if err := enqueueOperatorArtifactInputTx(ctx, tx, c.IncidentID, "captured_verdict_recorded", idempotencyKey, nil, id, now); err != nil { + return nil, nil, err + } if err := tx.Commit(); err != nil { return nil, nil, fmt.Errorf("store: commit verdict: %w", err) } diff --git a/internal/store/verdicts_test.go b/internal/store/verdicts_test.go index be18cbb..8dd6ac0 100644 --- a/internal/store/verdicts_test.go +++ b/internal/store/verdicts_test.go @@ -5,6 +5,7 @@ package store import ( "context" "errors" + "fmt" "testing" "time" ) @@ -212,3 +213,208 @@ func TestGoverningVerdict_DrillParity(t *testing.T) { t.Fatalf("drill read must see the drill verdict, got %+v, %v", v, err) } } + +// ---------------------------------------------------------------------- +// Task 8: atomic write-back — PersistVerdictCapture enqueues exactly one +// captured_verdict_recorded situation input (never an +// operator_annotation_recorded one, even though it also writes a matching +// incident_annotations row internally), under the same active/no-owner/ +// terminal-owner rules as annotations_test.go's InsertIncidentAnnotation +// coverage. situationOwnedIncident/situationIDForIncident/ +// terminalizeSituation/countOutboxRows are defined in annotations_test.go +// (same package). +// ---------------------------------------------------------------------- + +// TestPersistVerdictCapture_EnqueuesCapturedVerdictRecordedWhenOwnerActive +// proves Step 3's core contract and that the existing governing-verdict/ +// Triage effect is unchanged: LatestIncidentVerdict/GoverningVerdict still +// see the captured verdict exactly as before — the new outbox row is a pure +// addition, not a substitute for the verdict/annotation rows themselves. +func TestPersistVerdictCapture_EnqueuesCapturedVerdictRecordedWhenOwnerActive(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=verdict-active") + + v, ann, err := s.PersistVerdictCapture(ctx, VerdictCapture{ + IncidentID: id, Verdict: "correction", + Source: VerdictSourceHuman, LabelConfidence: 1, + ExpectationJSON: `{"must_not_conclude":["AZ outage"]}`, + AnnotationNote: "corrected: not AZ outage", + }) + if err != nil { + t.Fatalf("persist: %v", err) + } + + if n := countOutboxRows(t, s, id, "captured_verdict_recorded"); n != 1 { + t.Fatalf("captured_verdict_recorded inputs = %d, want 1", n) + } + // Never a separate operator_annotation_recorded input for the same + // event: PersistVerdictCapture's internal annotation write is not itself + // a plain-annotate write-back. + if n := countOutboxRows(t, s, id, "operator_annotation_recorded"); n != 0 { + t.Fatalf("operator_annotation_recorded inputs = %d, want 0 (captured verdict must not also enqueue an annotation input)", n) + } + var verdictID int64 + var status, journalState string + if err := s.db.QueryRowContext(ctx, ` + SELECT verdict_id, status, journal_state FROM situation_input_outbox + WHERE incident_id = ? AND kind = 'captured_verdict_recorded'`, id). + Scan(&verdictID, &status, &journalState); err != nil { + t.Fatal(err) + } + if verdictID != v.ID { + t.Fatalf("verdict_id = %d, want %d (the exact verdict this call persisted)", verdictID, v.ID) + } + if status != "pending" || journalState != "pending" { + t.Fatalf("status=%q journal_state=%q, want pending/pending", status, journalState) + } + + // Existing governing-verdict/Triage effect is unchanged. + latest, err := s.LatestIncidentVerdict(ctx, id) + if err != nil || latest == nil || latest.Version != 1 || latest.Verdict != "correction" { + t.Fatalf("latest verdict unaffected: %+v, %v", latest, err) + } + gov, err := s.GoverningVerdict(ctx, "service=verdict-active", false) + if err != nil || gov == nil || gov.IncidentID != id { + t.Fatalf("governing verdict unaffected: %+v, %v", gov, err) + } + if ann.Kind != "correction" { + t.Fatalf("matching annotation row unaffected: %+v", ann) + } +} + +// TestPersistVerdictCapture_NoEnqueueWithoutOwner mirrors +// TestInsertIncidentAnnotation_NoEnqueueWithoutOwner for the verdict path. +func TestPersistVerdictCapture_NoEnqueueWithoutOwner(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := readyIncident(t, s, "service=verdict-unowned") + + if _, _, err := s.PersistVerdictCapture(ctx, VerdictCapture{ + IncidentID: id, Verdict: "correction", + Source: VerdictSourceHuman, LabelConfidence: 1, + ExpectationJSON: `{}`, AnnotationNote: "note", + }); err != nil { + t.Fatalf("persist: %v", err) + } + if v, err := s.LatestIncidentVerdict(ctx, id); err != nil || v == nil { + t.Fatalf("verdict must still be visible: v=%+v err=%v", v, err) + } + if n := countOutboxRows(t, s, id, "captured_verdict_recorded"); n != 0 { + t.Fatalf("captured_verdict_recorded inputs = %d, want 0 (no owner at all)", n) + } +} + +// TestPersistVerdictCapture_NoEnqueueForTerminalOwner mirrors +// TestInsertIncidentAnnotation_NoEnqueueForTerminalOwner for the verdict +// path. +func TestPersistVerdictCapture_NoEnqueueForTerminalOwner(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=verdict-terminal") + situationID := situationIDForIncident(t, s, id) + terminalizeSituation(t, s, situationID, time.Now().UTC().Add(time.Hour)) + + if _, _, err := s.PersistVerdictCapture(ctx, VerdictCapture{ + IncidentID: id, Verdict: "correction", + Source: VerdictSourceHuman, LabelConfidence: 1, + ExpectationJSON: `{}`, AnnotationNote: "note", + }); err != nil { + t.Fatalf("persist: %v", err) + } + if v, err := s.LatestIncidentVerdict(ctx, id); err != nil || v == nil { + t.Fatalf("verdict must still be visible: v=%+v err=%v", v, err) + } + if n := countOutboxRows(t, s, id, "captured_verdict_recorded"); n != 0 { + t.Fatalf("captured_verdict_recorded inputs = %d, want 0 (owner already terminal at write time)", n) + } +} + +// TestPersistVerdictCapture_RetrySameVerdictIDIsIdempotent mirrors +// TestInsertIncidentAnnotation_RetrySameAnnotationIDIsIdempotent for the +// verdict path's own idempotency-key derivation. +func TestPersistVerdictCapture_RetrySameVerdictIDIsIdempotent(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=verdict-retry") + verdictID, err := insertVerdictRow(ctx, s, id, 1) + if err != nil { + t.Fatalf("seed verdict: %v", err) + } + idempotencyKey := fmt.Sprintf("captured-verdict:%d", verdictID) + + for i := 0; i < 2; i++ { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := enqueueOperatorArtifactInputTx(ctx, tx, id, "captured_verdict_recorded", idempotencyKey, nil, verdictID, time.Now().UTC()); err != nil { + t.Fatalf("enqueue attempt %d: %v", i, err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + if n := countOutboxRows(t, s, id, "captured_verdict_recorded"); n != 1 { + t.Fatalf("captured_verdict_recorded inputs after retry = %d, want 1 (idempotent replay)", n) + } +} + +// TestPersistVerdictCapture_R2RaceOwnerTerminalizesBeforeApply mirrors +// TestInsertIncidentAnnotation_R2RaceOwnerTerminalizesBeforeApply for the +// verdict path's full round trip through Task 2's already-implemented +// ApplySituationInput R2 handling. +func TestPersistVerdictCapture_R2RaceOwnerTerminalizesBeforeApply(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + id := situationOwnedIncident(t, s, "service=verdict-r2-race") + situationID := situationIDForIncident(t, s, id) + + v, _, err := s.PersistVerdictCapture(ctx, VerdictCapture{ + IncidentID: id, Verdict: "correction", + Source: VerdictSourceHuman, LabelConfidence: 1, + ExpectationJSON: `{}`, AnnotationNote: "note", + }) + if err != nil { + t.Fatalf("persist: %v", err) + } + before := getSituationByID(t, s, situationID) + + terminalizeSituation(t, s, situationID, time.Now().UTC().Add(time.Hour)) + + claim := claimOneInput(t, s, "input-worker", time.Now().UTC().Add(2*time.Hour)) + if err := s.ApplySituationInput(ctx, claim); err != nil { + t.Fatalf("apply artifact input to now-terminal owner: %v", err) + } + + after := getSituationByID(t, s, situationID) + if after.InputVersion != before.InputVersion { + t.Fatalf("input_version changed: before %d, after %d, want unchanged", before.InputVersion, after.InputVersion) + } + + var journalState string + var verdictID int64 + if err := s.db.QueryRowContext(ctx, ` + SELECT journal_state, verdict_id FROM situation_input_outbox + WHERE incident_id = ? AND kind = 'captured_verdict_recorded'`, id).Scan(&journalState, &verdictID); err != nil { + t.Fatal(err) + } + if journalState != "owner_terminal" { + t.Fatalf("journal_state = %q, want owner_terminal", journalState) + } + if verdictID != v.ID { + t.Fatalf("verdict_id = %d, want %d", verdictID, v.ID) + } + + var transitions int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, situationID).Scan(&transitions); err != nil { + t.Fatal(err) + } + if transitions != 0 { + t.Fatalf("situation_transitions for %s = %d, want 0 (owner_terminal is never journaled)", situationID, transitions) + } + + if latest, err := s.LatestIncidentVerdict(ctx, id); err != nil || latest == nil { + t.Fatalf("verdict must remain visible via Incident MCP/audit: latest=%+v err=%v", latest, err) + } +} diff --git a/skills/acutetriage/capture.go b/skills/acutetriage/capture.go index f5b9065..743c983 100644 --- a/skills/acutetriage/capture.go +++ b/skills/acutetriage/capture.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "github.com/alertint/alertint-agent/internal/notify" "github.com/alertint/alertint-agent/internal/store" ) @@ -123,11 +122,19 @@ type AnnotateResult struct { Demoted bool } -// Annotate stores a kind+note annotation, audits, and fans out the annotation -// event (Slack thread reply + stdout line). Notes speak to the next -// investigator only (channel split, ADR-0028 as amended): an annotation pulls -// no lever — no recall demotion, no marks floor. Machine effect belongs to -// verdict capture. The finding row is never touched. +// Annotate stores a kind+note annotation and audits it. Notes speak to the +// next investigator only (channel split, ADR-0028 as amended): an annotation +// pulls no lever — no recall demotion, no marks floor. Machine effect +// belongs to verdict capture. The finding row is never touched. +// +// Task 8: this no longer fans the event out to a notifier itself (the +// removed notifyAnnotation) — InsertIncidentAnnotation's own transaction now +// atomically enqueues a durable operator_annotation_recorded Situation input +// whenever the Incident belongs to a nonterminal Situation, and the +// Situation controller/notification worker (Task 6/7) is the sole surface +// that presents it. With no owning Situation, the annotation stays visible +// only through Incident MCP/audit — exactly like before, minus the direct +// Slack/stdout fan-out. func (e *CaptureEngine) Annotate(ctx context.Context, req AnnotateRequest) (*AnnotateResult, error) { if req.Kind != "correction" && req.Kind != "observation" { return nil, fmt.Errorf("acutetriage: annotate: kind %q not in {correction, observation} (confirmation is written by capture only)", req.Kind) @@ -158,29 +165,9 @@ func (e *CaptureEngine) Annotate(ctx context.Context, req AnnotateRequest) (*Ann return nil, fmt.Errorf("acutetriage: annotate: audit: %w", err) } } - e.notifyAnnotation(ctx, inc, req.Kind, req.Note, 0) return &AnnotateResult{AnnotationID: ann.ID, Demoted: false}, nil } -// notifyAnnotation fans the event out when the notifier supports it. -// Best-effort: a sink failure never fails the write that already landed. -func (e *CaptureEngine) notifyAnnotation(ctx context.Context, inc *store.Incident, kind, note string, verdictVersion int) { - sink, ok := e.sk.notifier.(interface { - OnAnnotation(ctx context.Context, ev notify.AnnotationEvent) error - }) - if !ok || e.sk.notifier == nil { - return - } - drill := false - if alerts, err := e.sk.st.GetIncidentAlerts(ctx, inc.ID); err == nil { - drill = isDrill(alerts) - } - _ = sink.OnAnnotation(ctx, notify.AnnotationEvent{ - IncidentID: inc.ID, GroupKey: inc.GroupKey, Kind: kind, Note: note, - VerdictVersion: verdictVersion, Drill: drill, - }) -} - // maxWidenQueries bounds one capture call's live widening fetches (D10). const maxWidenQueries = 10 @@ -250,7 +237,7 @@ func (e *CaptureEngine) CaptureVerdict(ctx context.Context, req CaptureRequest) verdictRow := latest needsPersist := latest == nil || latest.ExpectationJSON != expJSON || latest.Verdict != req.Verdict || len(newExprs) > 0 if needsPersist { - v, persistWarnings, err := e.persistCapture(ctx, req, exp, expJSON, inc, priorWidened, newExprs) + v, persistWarnings, err := e.persistCapture(ctx, req, exp, expJSON, priorWidened, newExprs) if err != nil { return nil, err } @@ -277,9 +264,14 @@ func (e *CaptureEngine) CaptureVerdict(ctx context.Context, req CaptureRequest) // persistCapture runs the persist phase (D7/D9): widen the not-yet-frozen // exprs live once, merge with what's already frozen, write the verdict + -// annotation + demotion atomically, audit, and fan out the annotation event. +// annotation + demotion atomically, and audit. Task 8: PersistVerdictCapture's +// own transaction now atomically enqueues a durable captured_verdict_recorded +// Situation input whenever the Incident belongs to a nonterminal Situation — +// this method no longer fans the event out to a notifier itself (the removed +// notifyAnnotation call); the Situation controller/notification worker owns +// presentation. func (e *CaptureEngine) persistCapture(ctx context.Context, req CaptureRequest, exp Expectation, expJSON string, - inc *store.Incident, priorWidened []VerificationQuery, newExprs []string, + priorWidened []VerificationQuery, newExprs []string, ) (*store.IncidentVerdict, []string, error) { fetched, warnings := e.widen(ctx, req.IncidentID, newExprs) merged := make([]VerificationQuery, 0, len(priorWidened)+len(fetched)) @@ -319,7 +311,6 @@ func (e *CaptureEngine) persistCapture(ctx context.Context, req CaptureRequest, return nil, nil, fmt.Errorf("acutetriage: capture: audit: %w", err) } } - e.notifyAnnotation(ctx, inc, req.Verdict, note, v.Version) return v, warnings, nil } diff --git a/skills/acutetriage/capture_test.go b/skills/acutetriage/capture_test.go index 3e6c984..8a9aa5b 100644 --- a/skills/acutetriage/capture_test.go +++ b/skills/acutetriage/capture_test.go @@ -7,6 +7,8 @@ import ( "encoding/json" "errors" "log/slog" + "net/http" + "net/http/httptest" "os" "path/filepath" "slices" @@ -98,7 +100,15 @@ func seedAnalyzedIncidentOnKey(t *testing.T, st *store.Store) store.Incident { return *got } -func TestAnnotate_CorrectionAuditsNotifiesNoDemote(t *testing.T) { +// TestAnnotate_CorrectionAuditsNoDemote proves Annotate's persist-only +// contract post-Task-8: the annotation lands, is audited, pulls no D7 lever, +// and — since Task 8 removed capture.go's own notifyAnnotation fan-out — +// never calls a notifier directly any more. Presentation is the Situation +// controller/notification worker's job (Task 6/7); this engine's own job is +// now limited to persisting the annotation and (situationOwnedIncident +// cases below) durably enqueuing the Situation input that makes it visible +// there. +func TestAnnotate_CorrectionAuditsNoDemote(t *testing.T) { st := newTestStore(t) inc := seedAnalyzedIncidentOnKey(t, st) sink := &fakeAnnotationSink{} @@ -129,8 +139,88 @@ func TestAnnotate_CorrectionAuditsNotifiesNoDemote(t *testing.T) { t.Fatalf("audit chain must verify: report=%+v err=%v", rep, err) } - if len(sink.events) != 1 || sink.events[0].Kind != "correction" || sink.events[0].Note != "not an AZ outage" { - t.Fatalf("annotation sink saw %+v", sink.events) + if len(sink.events) != 0 { + t.Fatalf("Annotate must never call a notifier directly (Task 8: the Situation controller/worker owns presentation); sink saw %+v", sink.events) + } +} + +// situationOwnedIncident builds one real Incident and durably attaches it to +// a brand-new active Situation the same way the durable pipeline does — +// InsertIncident, a queued incident_created situation_input_outbox row, +// ClaimSituationInputs, ApplySituationInput (mirrors internal/mcp's +// seedSituationForMCP and internal/store's own situationOwnedIncident) — +// never an INSERT INTO situations by hand. Task 8's write-back enqueue +// (InsertIncidentAnnotation/PersistVerdictCapture, called through +// CaptureEngine here) only reaches situation_input_outbox when the target +// Incident currently belongs to a nonterminal Situation, so the tests below +// that assert on that enqueue call this first. +func situationOwnedIncident(t *testing.T, st *store.Store, groupKey string) string { + t.Helper() + ctx := context.Background() + now := time.Now().UTC() + incidentID := uuid.NewString() + if err := st.InsertIncident(ctx, store.Incident{ + ID: incidentID, GroupKey: groupKey, FirstAlertAt: now, LastAlertAt: now, ReadyAt: now.Add(time.Minute), + }); err != nil { + t.Fatalf("insert incident: %v", err) + } + if err := st.MarkIncidentReady(ctx, incidentID); err != nil { + t.Fatalf("mark incident ready: %v", err) + } + inputID := "input-" + incidentID + if _, err := st.DB().ExecContext(ctx, ` + INSERT INTO situation_input_outbox (id, idempotency_key, incident_id, kind, group_key, occurred_at, status) + VALUES (?, ?, ?, 'incident_created', ?, ?, 'pending')`, + inputID, "idem:"+inputID, incidentID, groupKey, now.Format(time.RFC3339Nano)); err != nil { + t.Fatalf("seed situation input: %v", err) + } + claims, err := st.ClaimSituationInputs(ctx, "test-worker", now, time.Minute, 1) + if err != nil || len(claims) != 1 { + t.Fatalf("claim seed situation input: claims=%d err=%v", len(claims), err) + } + if err := st.ApplySituationInput(ctx, claims[0]); err != nil { + t.Fatalf("apply seed situation input: %v", err) + } + return incidentID +} + +// countSituationInputKind counts situation_input_outbox rows of kind for +// incidentID — the write-back enqueue's own visible effect. +func countSituationInputKind(t *testing.T, st *store.Store, incidentID, kind string) int { + t.Helper() + var n int + if err := st.DB().QueryRowContext(context.Background(), ` + SELECT COUNT(*) FROM situation_input_outbox WHERE incident_id = ? AND kind = ?`, incidentID, kind).Scan(&n); err != nil { + t.Fatalf("count situation inputs: %v", err) + } + return n +} + +// TestAnnotate_NoDirectNotifyEnqueuesOperatorAnnotationRecordedInput proves +// Task 8's Step 5 cutover end to end from the CaptureEngine's own entry +// point: Annotate never calls a notifier directly (no direct Slack call), +// and — because the target Incident belongs to an active Situation — the +// durable write-back enqueues exactly one operator_annotation_recorded +// situation_input_outbox row (the "one attributable Situation journal" +// input a later controller reconciliation cycle folds into a Transition). +func TestAnnotate_NoDirectNotifyEnqueuesOperatorAnnotationRecordedInput(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + incidentID := situationOwnedIncident(t, st, "service=annotate-situation") + sink := &fakeAnnotationSink{} + eng := acutetriage.NewCaptureEngine(skillForCapture(t, st, sink)) + + if _, err := eng.Annotate(ctx, acutetriage.AnnotateRequest{ + IncidentID: incidentID, Kind: "observation", Note: "fyi", + }); err != nil { + t.Fatal(err) + } + + if len(sink.events) != 0 { + t.Fatalf("Annotate must never call a notifier directly; sink saw %+v", sink.events) + } + if n := countSituationInputKind(t, st, incidentID, "operator_annotation_recorded"); n != 1 { + t.Fatalf("operator_annotation_recorded situation inputs = %d, want 1", n) } } @@ -240,6 +330,37 @@ func TestCaptureVerdict_PersistPhase(t *testing.T) { } } +// TestCaptureVerdict_NoDirectNotifyEnqueuesCapturedVerdictRecordedInput +// mirrors TestAnnotate_NoDirectNotifyEnqueuesOperatorAnnotationRecordedInput +// for the CaptureVerdict persist phase: no direct notifier call, and exactly +// one captured_verdict_recorded situation input when the Incident belongs to +// an active Situation. +func TestCaptureVerdict_NoDirectNotifyEnqueuesCapturedVerdictRecordedInput(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + incidentID := situationOwnedIncident(t, st, "service=verdict-situation") + insertTestAlert(t, st, ctx, incidentID, "fp-"+incidentID, map[string]string{"alertname": "TargetDown"}) + if err := st.SaveIncidentOutput(ctx, incidentID, `{"analysis_name":"x","overall_issue":"y"}`, "x", "y", 0.7, "{}"); err != nil { + t.Fatalf("save incident output: %v", err) + } + sink := &fakeAnnotationSink{} + eng := acutetriage.NewCaptureEngine(skillForCaptureWithProm(t, st, sink, promHealthy(t))) + + if _, err := eng.CaptureVerdict(ctx, acutetriage.CaptureRequest{ + IncidentID: incidentID, Verdict: "correction", + Expectation: json.RawMessage(`{"must_not_conclude":["AZ outage"]}`), + }); err != nil { + t.Fatal(err) + } + + if len(sink.events) != 0 { + t.Fatalf("CaptureVerdict must never call a notifier directly; sink saw %+v", sink.events) + } + if n := countSituationInputKind(t, st, incidentID, "captured_verdict_recorded"); n != 1 { + t.Fatalf("captured_verdict_recorded situation inputs = %d, want 1", n) + } +} + func TestCaptureVerdict_RepeatCallSkipsPersist(t *testing.T) { st := newTestStore(t) ctx := context.Background() @@ -897,25 +1018,30 @@ func TestCaptureEngineCloseJoinsGrading(t *testing.T) { } } -// blockingSink holds the annotation fan-out (the last step of the persist -// phase, strictly before grading) until released. -type blockingSink struct { - started chan struct{} - release chan struct{} - once sync.Once -} - -func (b *blockingSink) Name() string { return "blocking" } -func (b *blockingSink) Notify(context.Context, notify.Finding) error { return nil } -func (b *blockingSink) OnAnnotation(context.Context, notify.AnnotationEvent) error { - b.once.Do(func() { close(b.started) }) - <-b.release - return nil +// blockingWidenProm builds a Prometheus test server whose query handler +// blocks until release is closed, signaling started exactly once first. It +// is TestCaptureEngineCloseWaitsForThePersistPhase's mid-persist observation +// point: since Task 8 removed capture.go's own notifyAnnotation fan-out (the +// former last step of the persist phase, strictly before grading), the live +// widen() fetch is the only externally-blockable step left inside +// persistCapture, so this replaces the former blockingSink. +func blockingWidenProm(t *testing.T, started, release chan struct{}) *promclient.Client { + t.Helper() + var once sync.Once + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + once.Do(func() { close(started) }) + <-release + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(vectorValue3)) + })) + t.Cleanup(srv.Close) + return promclient.NewClient(promclient.Config{BaseURL: srv.URL, TimeoutSeconds: 30}) } // TestCaptureEngineCloseWaitsForThePersistPhase: the engine's join must cover // the WHOLE Captured-verdict operation, not just its grade. A handler still -// persisting (store writes, audit, widening, fan-out) is invisible to an +// persisting (store writes, audit, widening) is invisible to an // http.Server.Shutdown that has given up, so if Close could return while one // is in the persist phase, the owner would close the store underneath it — // and the runner's final pass would run with a producer about to enter @@ -925,15 +1051,18 @@ func (b *blockingSink) OnAnnotation(context.Context, notify.AnnotationEvent) err func TestCaptureEngineCloseWaitsForThePersistPhase(t *testing.T) { st := newTestStore(t) ctx := context.Background() - prom := promHealthy(t) - inc := seedGradableIncident(t, st, prom) + seedProm := promHealthy(t) + inc := seedGradableIncident(t, st, seedProm) tr, err := llmhealth.New(ctx, st, llmhealth.Options{}) if err != nil { t.Fatal(err) } - cfg := verifyConfig(prom) + started := make(chan struct{}) + release := make(chan struct{}) + cfg := verifyConfig(blockingWidenProm(t, started, release)) + cfg.Verification.QueryTimeoutSeconds = 30 cfg.Health = tr - sink := &blockingSink{started: make(chan struct{}), release: make(chan struct{})} + sink := &fakeAnnotationSink{} gradeLLM := &blockingLLM{started: make(chan struct{})} eng := acutetriage.NewCaptureEngine(acutetriage.New(cfg, st, gradeLLM, audit.New(st.DB()), notify.NewMulti(nil, sink), nil)) @@ -945,11 +1074,12 @@ func TestCaptureEngineCloseWaitsForThePersistPhase(t *testing.T) { go func() { res, err := eng.CaptureVerdict(ctx, acutetriage.CaptureRequest{ IncidentID: inc.ID, Verdict: "correction", - Expectation: json.RawMessage(`{"must_mention":["worker-14"]}`), + Expectation: json.RawMessage(`{"must_mention":["worker-14"]}`), + WidenQueries: []string{"node_network_up"}, }) got <- outcome{res, err} }() - <-sink.started // the operation is mid-persist, before enterGrade + <-started // the operation is mid-persist (a live widen fetch), before enterGrade closed := make(chan error, 1) go func() { @@ -963,7 +1093,7 @@ func TestCaptureEngineCloseWaitsForThePersistPhase(t *testing.T) { case <-time.After(150 * time.Millisecond): } - close(sink.release) + close(release) select { case err := <-closed: if err != nil { From 3f80349e6db4d81f69efb628f742c28ea3455f4f Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 08:21:16 +0300 Subject: [PATCH 15/31] feat(runtime): expose Situation history delivery Signed-off-by: ernescz --- cmd/alertint/main.go | 31 +- cmd/alertint/main_test.go | 61 ++ cmd/alertint/situation_controller.go | 47 +- cmd/alertint/situation_controller_test.go | 45 +- cmd/alertint/situation_foundation.go | 29 + cmd/alertint/situation_foundation_test.go | 116 ++++ cmd/alertint/situation_notifications.go | 362 ++++++++++ cmd/alertint/situation_notifications_test.go | 334 +++++++++ docs/integrations/mcp-clients.md | 4 +- internal/audit/audit.go | 126 ++++ internal/audit/audit_test.go | 106 +++ internal/mcp/docs_drift_test.go | 4 + internal/mcp/server.go | 6 + internal/mcp/server_situations.go | 377 +++++++++- internal/mcp/server_situations_test.go | 341 +++++++++ internal/notify/stdout/situation.go | 618 +++++++++++++++++ internal/notify/stdout/situation_test.go | 649 ++++++++++++++++++ internal/situation/controller.go | 119 ++++ internal/situation/notification_worker.go | 135 +++- .../situation/notification_worker_test.go | 232 +++++++ internal/situation/telemetry.go | 118 ++++ internal/situation/telemetry_test.go | 152 ++++ internal/store/situation_history.go | 11 + internal/store/situation_startup.go | 103 +++ internal/store/situation_transition_stream.go | 243 +++++++ .../store/situation_transition_stream_test.go | 380 ++++++++++ internal/store/situation_views.go | 238 +++++++ 27 files changed, 4946 insertions(+), 41 deletions(-) create mode 100644 internal/notify/stdout/situation.go create mode 100644 internal/notify/stdout/situation_test.go create mode 100644 internal/store/situation_startup.go create mode 100644 internal/store/situation_transition_stream.go create mode 100644 internal/store/situation_transition_stream_test.go diff --git a/cmd/alertint/main.go b/cmd/alertint/main.go index 83ae119..3e5801d 100644 --- a/cmd/alertint/main.go +++ b/cmd/alertint/main.go @@ -451,11 +451,22 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { // the Correlator itself (cor, constructed above) receives no LLM // dependency of its own — corCfg/correlator.New's signature carries none, // and its only path to Acute Triage is via incidentSink{skill: skill}. - crt, err := buildControllerRuntime(st, llmClient, llmHealth, skill, cfg.Situations, owner, auditor, logger) + crt, err := buildControllerRuntime(st, llmClient, llmHealth, skill, cfg.Situations, + cfg.Notify.Slack.MinSeverity, owner, auditor, logger) if err != nil { return err } + // The Situation notification runtime (Plan 3 Task 9): the single + // reachable Situation Slack writer (present only when Situation Slack is + // actually configured — buildSituationNotificationRuntime) plus the + // stdout Transition-stream worker, which always runs because the + // authoritative outward state stream is not Slack-gated. Its own + // startup-only recovery pass (spec.md startup steps 2-6) runs after + // Plan 2's controller recovery and before the Correlator, and both its + // workers stop LAST, outside the shutdown drain rounds (R6). + nrt := buildSituationNotificationRuntime(cfg, st, auditor, owner, logger) + // Probe enabled integrations in the background: quickly (with backoff) // while one is failing — at startup a co-deployed dependency may still // be booting — then at a steady pace, logging losses and recoveries. @@ -477,6 +488,15 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { backfillAndRecoverControllerWork: func(ctx context.Context) error { return runControllerRecovery(ctx, crt, logger) }, + // Plan 3 Task 9: recover abandoned notification/stream claims, + // schedule Situations whose durable history is missing or whose root + // projection is stale, validate the Slack configuration and record + // its generation, reactivate configuration-blocked effects, and + // resume an interrupted gap replay — all startup-only and all + // publication-free, exactly like the two recovery passes above. + recoverNotificationWork: func(ctx context.Context) error { + return runNotificationRecovery(ctx, nrt, logger) + }, // SetAuditor is wired here — between reconstruct and cor.Start, never // before — because it is synchronously reachable from ApplyDelivery's // durable-dispatch path (a queued retry attach appends @@ -499,8 +519,9 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { cor.SetAuditor(auditor) return cor.Start(ctx) }, - startWorkers: rt.Start, - startControllerWorkers: crt.Start, + startWorkers: rt.Start, + startControllerWorkers: crt.Start, + startNotificationWorkers: nrt.Start, startReceivers: func() error { var err error recvSrv, recvErrCh, err = startReceivers(cfg, st, auditor, healthReg, llmHealth, rt.WakeDispatch, logger) @@ -551,6 +572,10 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { drainControllerWork: crt.Drain, stopControllerWorkers: crt.Stop, stopWorkers: rt.Stop, + // R6, last and outside the drain rounds: one bounded final delivery + // and stdout pass under the shutdown context, then claim release. An + // unreachable Slack delays the pass, it never holds the process. + stopNotificationWorkers: nrt.Stop, } if err := stopSeq.run(shutdownCtx); err != nil { logger.Error("situation foundation shutdown failed", slog.String("err", err.Error())) diff --git a/cmd/alertint/main_test.go b/cmd/alertint/main_test.go index 8cfb67f..29bdb8b 100644 --- a/cmd/alertint/main_test.go +++ b/cmd/alertint/main_test.go @@ -235,3 +235,64 @@ func TestBuildLogger_Precedence(t *testing.T) { }) } } + +// TestSituationNotificationRuntimeAssemblyGatesSlackOnConfiguration proves +// Plan 3 Task 9's assembly rule: exactly one Situation Slack writer is +// constructed, and only when Situation Slack is actually usable. With Slack +// off (or with no resolvable token) no Slack credential is constructed on +// this path at all, while the stdout Transition-stream worker always runs — +// the authoritative outward state stream is never Slack-gated. +func TestSituationNotificationRuntimeAssemblyGatesSlackOnConfiguration(t *testing.T) { + st := newTestFoundationStore(t) + logger := slog.New(slog.DiscardHandler) + + off := config.Defaults() + off.Notify.Slack.Enabled = false + nrt := buildSituationNotificationRuntime(&off, st, nil, "owner-off", logger) + if nrt.worker != nil || nrt.probe != nil { + t.Error("a Slack notification worker was constructed with notify.slack disabled") + } + if nrt.stream == nil { + t.Error("the stdout Transition-stream worker must run regardless of Slack configuration") + } + + noToken := config.Defaults() + noToken.Notify.Slack.Enabled = true + noToken.Notify.Slack.Channel = "#alerts" + noToken.Notify.Slack.BotTokenEnv = "ALERTINT_TEST_SITUATION_SLACK_ABSENT" + if nrt := buildSituationNotificationRuntime(&noToken, st, nil, "owner-untokened", logger); nrt.worker != nil { + t.Error("a Slack notification worker was constructed with no resolvable bot token") + } + + on := config.Defaults() + on.Notify.Slack.Enabled = true + on.Notify.Slack.Channel = "#alerts" + on.Notify.Slack.BotTokenEnv = "ALERTINT_TEST_SITUATION_SLACK_TOKEN" + t.Setenv("ALERTINT_TEST_SITUATION_SLACK_TOKEN", "xoxb-test") + nrt = buildSituationNotificationRuntime(&on, st, nil, "owner-on", logger) + if nrt.worker == nil || nrt.probe == nil { + t.Fatal("no Situation Slack notification worker was constructed with Slack fully configured") + } + if _, ok := nrt.worker.(*situation.NotificationWorker); !ok { + t.Fatalf("Situation Slack writer is %T, want exactly one *situation.NotificationWorker", nrt.worker) + } +} + +// TestSituationNotificationRuntimeIsTheOnlySituationSlackWriterInAssembly is +// a source-text scan proving cmd/alertint constructs a Slack API credential +// on exactly two paths: buildNotifier's ADR-0042/ADR-0046 System-message +// notifier, and buildSituationSlackWorker's Situation deliverer. +func TestSituationNotificationRuntimeIsTheOnlySituationSlackWriterInAssembly(t *testing.T) { + constructors := 0 + for _, name := range []string{"main.go", "situation_notifications.go", "situation_controller.go", "situation_foundation.go"} { + src, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + constructors += strings.Count(string(src), "notifyslack.New(") + strings.Count(string(src), "slack.NewClient(") + } + if constructors != 2 { + t.Fatalf("cmd/alertint constructs %d Slack clients in its assembly files, want exactly 2 "+ + "(the System-message notifier and the Situation deliverer)", constructors) + } +} diff --git a/cmd/alertint/situation_controller.go b/cmd/alertint/situation_controller.go index dfe3f9b..d7319b8 100644 --- a/cmd/alertint/situation_controller.go +++ b/cmd/alertint/situation_controller.go @@ -14,6 +14,7 @@ import ( "github.com/alertint/alertint-agent/internal/llm" "github.com/alertint/alertint-agent/internal/llmhealth" "github.com/alertint/alertint-agent/internal/situation" + "github.com/alertint/alertint-agent/internal/situation/model" "github.com/alertint/alertint-agent/internal/store" "github.com/alertint/alertint-agent/skills/acutetriage" ) @@ -73,6 +74,7 @@ func newControllerRuntime( assessClient situation.AssessmentClient, skill *acutetriage.Skill, cfg config.SituationsConfig, + slackMinSeverity string, owner string, auditSink situation.AuditSink, logger *slog.Logger, @@ -80,7 +82,7 @@ func newControllerRuntime( if strings.TrimSpace(owner) == "" { panic("cmd/alertint: controller runtime requires a non-empty owner") } - controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, owner) + controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, slackInterruptionFloor(slackMinSeverity), owner) worker := situation.NewControllerWorker(st, st, assessClient, controllerCfg, workerCfg, nil, auditSink, logger) @@ -116,6 +118,7 @@ func buildControllerRuntime( llmHealth *llmhealth.Tracker, skill *acutetriage.Skill, cfg config.SituationsConfig, + slackMinSeverity string, owner string, auditSink situation.AuditSink, logger *slog.Logger, @@ -124,7 +127,7 @@ func buildControllerRuntime( if err != nil { return nil, fmt.Errorf("situation controller: %w", err) } - crt := newControllerRuntime(st, assessClient, skill, cfg, owner, auditSink, logger) + crt := newControllerRuntime(st, assessClient, skill, cfg, slackMinSeverity, owner, auditSink, logger) crt.SetDependencyRecoveryWaker(llmHealthDependencyWaker{tracker: llmHealth, st: st}) crt.SetAssessmentHealthObserver(llmHealthAssessmentObserver{tracker: llmHealth}) return crt, nil @@ -142,8 +145,20 @@ func buildControllerRuntime( // PollingIntervalSeconds has no config.SituationsConfig source (no polling // connector exists in this build) and is left at its zero-value default — // see ControllerConfig's own doc comment. -func situationsConfigToControllerConfig(cfg config.SituationsConfig, owner string) (situation.ControllerConfig, situation.ControllerWorkerConfig) { +// +// Plan 3 Task 9 adds the two publication-policy fields Task 5 declared but +// left unwired: notify.slack.min_severity -> ControllerConfig.SlackFloor +// (through slackInterruptionFloor, since the Situation path reads that +// setting as an Interruption-priority floor, never as Alert severity) and +// situations.slack.repage_cooldown_seconds -> ControllerConfig. +// RepageCooldown. Left at their zero values they would silently mean "no +// floor" and "the built-in 900s default", so an operator who configured +// either would have been ignored. +func situationsConfigToControllerConfig(cfg config.SituationsConfig, slackFloor model.InterruptionPriority, + owner string) (situation.ControllerConfig, situation.ControllerWorkerConfig) { controllerCfg := situation.ControllerConfig{ + SlackFloor: slackFloor, + RepageCooldown: time.Duration(cfg.Slack.RepageCooldownSeconds) * time.Second, Cadence: situation.CadenceTempo{ Fast: time.Duration(cfg.Cadence.FastSeconds) * time.Second, Normal: time.Duration(cfg.Cadence.NormalSeconds) * time.Second, @@ -170,6 +185,32 @@ func situationsConfigToControllerConfig(cfg config.SituationsConfig, owner strin return controllerCfg, workerCfg } +// slackInterruptionFloor maps the operator's existing notify.slack. +// min_severity setting onto the Situation path's minimum Interruption +// priority (spec.md "Publication authority and Interruption priority": the +// setting keeps its accepted low|medium|high values but is compared with +// Interruption priority ONLY — never with Alert or model severity). +// +// An empty or unrecognized value maps to the empty floor, which +// situation.MeetsSlackFloor treats as "no floor": a setting this build does +// not understand must never silently become a stricter one that withholds +// operator history. "critical" is a valid Interruption priority with no +// min_severity equivalent, so this compatibility setting can never select +// it (internal/config's own +// TestNotifySlackMinSeverityIsInterruptionPriorityFloor pins the value set). +func slackInterruptionFloor(minSeverity string) model.InterruptionPriority { + switch strings.ToLower(strings.TrimSpace(minSeverity)) { + case string(model.InterruptionLow): + return model.InterruptionLow + case string(model.InterruptionMedium): + return model.InterruptionMedium + case string(model.InterruptionHigh): + return model.InterruptionHigh + default: + return "" + } +} + // controllerRecovery is the startup-only recovery/backfill pass's report, // for logReconstructionReport's own sibling logging call. type controllerRecovery struct { diff --git a/cmd/alertint/situation_controller_test.go b/cmd/alertint/situation_controller_test.go index c0a3e9c..ca7d0d2 100644 --- a/cmd/alertint/situation_controller_test.go +++ b/cmd/alertint/situation_controller_test.go @@ -12,6 +12,7 @@ import ( "github.com/alertint/alertint-agent/internal/llm" "github.com/alertint/alertint-agent/internal/llmhealth" "github.com/alertint/alertint-agent/internal/situation" + "github.com/alertint/alertint-agent/internal/situation/model" "github.com/alertint/alertint-agent/internal/store" ) @@ -26,7 +27,7 @@ func TestSituationControllerRuntimePanicsOnEmptyOwner(t *testing.T) { t.Fatal("expected a panic for an empty owner") } }() - newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, " ", nil, nil) + newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", " ", nil, nil) } // TestSituationsConfigToControllerConfigMapsEveryField pins Task 8's own @@ -53,7 +54,18 @@ func TestSituationsConfigToControllerConfigMapsEveryField(t *testing.T) { JitterPercent: 21, }, } - controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, "owner-1") + cfg.Slack = config.SituationSlackConfig{RepageCooldownSeconds: 600} + controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, model.InterruptionHigh, "owner-1") + + // Plan 3 Task 9: the two publication-policy fields Task 5 added but left + // unwired must carry real operator configuration, not their zero values + // (which would silently mean "no floor" and the 900s default). + if controllerCfg.SlackFloor != model.InterruptionHigh { + t.Fatalf("SlackFloor = %q, want the configured notify.slack.min_severity floor", controllerCfg.SlackFloor) + } + if controllerCfg.RepageCooldown != 600*time.Second { + t.Fatalf("RepageCooldown = %v, want 600s from situations.slack.repage_cooldown_seconds", controllerCfg.RepageCooldown) + } if controllerCfg.MaxL2CallsPerAttempt != 2 || controllerCfg.MaxWorkAttemptsPerInput != 5 { t.Fatalf("controllerCfg budgets = %+v", controllerCfg) @@ -92,6 +104,29 @@ func TestSituationsConfigToControllerConfigMapsEveryField(t *testing.T) { } } +// TestSituationControllerRuntimeSlackFloorMapsMinSeverity pins the +// compatibility reinterpretation spec.md states explicitly: notify.slack. +// min_severity keeps its existing low|medium|high values, but the Situation +// path reads the selected value ONLY as a minimum deterministic Interruption +// priority — never as Alert or model severity. An unset or unrecognized +// value is "no floor", never a silently stricter one. +func TestSituationControllerRuntimeSlackFloorMapsMinSeverity(t *testing.T) { + for _, tc := range []struct { + minSeverity string + want model.InterruptionPriority + }{ + {"low", model.InterruptionLow}, + {"medium", model.InterruptionMedium}, + {"high", model.InterruptionHigh}, + {"", ""}, + {"URGENT", ""}, + } { + if got := slackInterruptionFloor(tc.minSeverity); got != tc.want { + t.Errorf("slackInterruptionFloor(%q) = %q, want %q", tc.minSeverity, got, tc.want) + } + } +} + // ---------------------------------------------------------------------- // controllerRuntime.RecoverAndBackfill / Start / Drain / Stop, against a // real (empty) store — proves the plumbing runs cleanly with nothing due, @@ -100,7 +135,7 @@ func TestSituationsConfigToControllerConfigMapsEveryField(t *testing.T) { func TestSituationControllerRuntimeRecoverAndBackfillOnEmptyStoreIsANoOp(t *testing.T) { st := newTestFoundationStore(t) - rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "test-owner", nil, nil) + rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "test-owner", nil, nil) report, err := rt.RecoverAndBackfill(context.Background(), time.Now().UTC()) if err != nil { @@ -115,7 +150,7 @@ func TestSituationControllerRuntimeRecoverAndBackfillOnEmptyStoreIsANoOp(t *test func TestSituationControllerRuntimeStartDrainStop(t *testing.T) { st := newTestFoundationStore(t) cfg := config.SituationsConfig{ReconcilePollSeconds: 3600, LeaseSeconds: 300, HeartbeatSeconds: 30} - rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, cfg, "test-owner", nil, nil) + rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, cfg, "", "test-owner", nil, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -503,7 +538,7 @@ func TestSituationControllerRuntimeRecoverAndBackfillAuditsStartupHorizonExhaust } audit := &fakeControllerRuntimeAuditSink{} - rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "test-owner", audit, nil) + rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "test-owner", audit, nil) report, err := rt.RecoverAndBackfill(context.Background(), now) if err != nil { diff --git a/cmd/alertint/situation_foundation.go b/cmd/alertint/situation_foundation.go index 8bc7aac..4ddfb85 100644 --- a/cmd/alertint/situation_foundation.go +++ b/cmd/alertint/situation_foundation.go @@ -210,9 +210,11 @@ func (r *foundationRuntime) WakeDispatch() { type foundationSequence struct { reconstruct func(ctx context.Context) error backfillAndRecoverControllerWork func(ctx context.Context) error + recoverNotificationWork func(ctx context.Context) error startCorrelator func(ctx context.Context) error startWorkers func(ctx context.Context) startControllerWorkers func(ctx context.Context) + startNotificationWorkers func(ctx context.Context) startReceivers func() error } @@ -225,6 +227,11 @@ func (f foundationSequence) run(ctx context.Context) error { return err } } + if f.recoverNotificationWork != nil { + if err := f.recoverNotificationWork(ctx); err != nil { + return err + } + } if err := f.startCorrelator(ctx); err != nil { return err } @@ -232,6 +239,9 @@ func (f foundationSequence) run(ctx context.Context) error { if f.startControllerWorkers != nil { f.startControllerWorkers(ctx) } + if f.startNotificationWorkers != nil { + f.startNotificationWorkers(ctx) + } return f.startReceivers() } @@ -286,6 +296,20 @@ type foundationStopSequence struct { drainControllerWork func(ctx context.Context) (int, error) stopControllerWorkers func(ctx context.Context) error stopWorkers func(ctx context.Context) error + // stopNotificationWorkers is Plan 3's own final stage (R6). It runs + // LAST — after every producer of durable history has stopped — and + // deliberately outside drainToQuiescence: the Situation notification + // worker and the stdout Transition-stream worker consume history that + // is already committed, so they owe shutdown exactly one bounded final + // pass under ctx followed by the release of every claim they still + // hold, never a share of the drain rounds. An unreachable Slack is the + // reason: joining the rounds would let an external outage spin the loop + // to maxShutdownDrainRounds and hold the whole process open, while a + // stage bounded by the shutdown context simply ends and leaves its + // committed intents pending for the next startup to reclaim. May be nil + // (no Plan 3 notification runtime composed in), in which case the + // sequence ends at stopWorkers exactly as it did before Plan 3. + stopNotificationWorkers func(ctx context.Context) error } // maxShutdownDrainRounds bounds foundationStopSequence's drain loop. @@ -308,6 +332,11 @@ func (f foundationStopSequence) run(ctx context.Context) error { if err := f.stopWorkers(ctx); err != nil { errs = append(errs, err) } + if f.stopNotificationWorkers != nil { + if err := f.stopNotificationWorkers(ctx); err != nil { + errs = append(errs, err) + } + } return errors.Join(errs...) } diff --git a/cmd/alertint/situation_foundation_test.go b/cmd/alertint/situation_foundation_test.go index 39b29ee..069e9c1 100644 --- a/cmd/alertint/situation_foundation_test.go +++ b/cmd/alertint/situation_foundation_test.go @@ -76,6 +76,14 @@ func tracedFoundationSequence(tr *tracer) foundationSequence { tr.add("enforce_triage_startup_horizon") return nil }, + recoverNotificationWork: func(context.Context) error { + tr.add("recover_notification_claims") + tr.add("schedule_situations_missing_first_transition") + tr.add("validate_slack_configuration") + tr.add("reactivate_configuration_blocked") + tr.add("supersede_stale_roots_and_resume_replay") + return nil + }, startCorrelator: func(context.Context) error { tr.add("start_correlator") return nil @@ -88,6 +96,10 @@ func tracedFoundationSequence(tr *tracer) foundationSequence { tr.add("start_controller_worker") tr.add("start_triage_worker") }, + startNotificationWorkers: func(context.Context) { + tr.add("start_notification_worker") + tr.add("start_transition_stream_worker") + }, startReceivers: func() error { tr.add("start_receivers") return nil @@ -119,11 +131,18 @@ func TestFoundationSequenceOrdersEveryPhase(t *testing.T) { "recover_interrupted_assessment_calls", "recover_interrupted_triage_attempts", "enforce_triage_startup_horizon", + "recover_notification_claims", + "schedule_situations_missing_first_transition", + "validate_slack_configuration", + "reactivate_configuration_blocked", + "supersede_stale_roots_and_resume_replay", "start_correlator", "start_input_worker", "start_dispatch_worker", "start_controller_worker", "start_triage_worker", + "start_notification_worker", + "start_transition_stream_worker", "start_receivers", } assertTrace(t, tr.snapshot(), want) @@ -169,6 +188,9 @@ func TestFoundationSequenceCorrelatorStartErrorPreventsReceiversStarting(t *test "recover_leases", "drain_deliveries", "drain_inputs", "reconstruct_incidents", "triage_migration_backfill", "recover_interrupted_assessment_calls", "recover_interrupted_triage_attempts", "enforce_triage_startup_horizon", + "recover_notification_claims", "schedule_situations_missing_first_transition", + "validate_slack_configuration", "reactivate_configuration_blocked", + "supersede_stale_roots_and_resume_replay", "start_correlator", } assertTrace(t, tr.snapshot(), want) @@ -237,6 +259,11 @@ func TestFoundationStopSequenceOrdersAllPhases(t *testing.T) { tr.add("stop_input_worker") return nil }, + stopNotificationWorkers: func(context.Context) error { + tr.add("stop_transition_stream_worker") + tr.add("stop_notification_worker") + return nil + }, } if err := seq.run(context.Background()); err != nil { @@ -247,10 +274,99 @@ func TestFoundationStopSequenceOrdersAllPhases(t *testing.T) { "drain_foundation_work", "drain_controller_work", "stop_triage_worker", "stop_controller_worker", "stop_dispatch_worker", "stop_input_worker", + "stop_transition_stream_worker", "stop_notification_worker", } assertTrace(t, tr.snapshot(), want) } +// TestFoundationStopSequenceNeverDrainsNotificationWorkers proves R6 +// structurally: the notification and stdout-stream workers are consumers of +// already-committed history, so they are stopped ONCE, after the foundation +// workers, and never take part in a single drain round. A Slack outage that +// makes every delivery attempt fail must not be able to keep the bounded +// drain loop spinning to its round cap. +func TestFoundationStopSequenceNeverDrainsNotificationWorkers(t *testing.T) { + tr := &tracer{} + notificationStops := 0 + drainRounds := 0 + seq := foundationStopSequence{ + stopReceivers: func() error { return nil }, + stopCorrelator: func() {}, + drainFoundationWork: func(context.Context) (int, error) { + drainRounds++ + tr.add("drain_foundation_work") + if drainRounds == 1 { + return 1, nil + } + return 0, nil + }, + drainControllerWork: func(context.Context) (int, error) { + tr.add("drain_controller_work") + return 0, nil + }, + stopControllerWorkers: func(context.Context) error { return nil }, + stopWorkers: func(context.Context) error { return nil }, + stopNotificationWorkers: func(context.Context) error { + notificationStops++ + tr.add("stop_notification_workers") + return nil + }, + } + if err := seq.run(context.Background()); err != nil { + t.Fatalf("run: %v", err) + } + if notificationStops != 1 { + t.Fatalf("notification workers stopped %d times, want exactly 1", notificationStops) + } + got := tr.snapshot() + if got[len(got)-1] != "stop_notification_workers" { + t.Fatalf("trace = %v, want stop_notification_workers last", got) + } + for _, phase := range got[:len(got)-1] { + if phase == "stop_notification_workers" { + t.Fatalf("trace = %v, want the notification stop to appear exactly once, at the end", got) + } + } +} + +// TestFoundationStopSequenceSurvivesASlackOutageAtShutdown proves the +// R6 guarantee that matters operationally: a notification stop whose one +// bounded final pass is still blocked on an unreachable Slack ends when the +// shutdown context does, and the sequence itself returns rather than +// hanging. The stage reports the context error (its committed intents stay +// pending, to be reclaimed at the next startup) and every earlier stage has +// already run. +func TestFoundationStopSequenceSurvivesASlackOutageAtShutdown(t *testing.T) { + tr := &tracer{} + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + seq := foundationStopSequence{ + stopReceivers: func() error { tr.add("stop_receivers"); return nil }, + stopCorrelator: func() { tr.add("stop_correlator") }, + stopWorkers: func(context.Context) error { tr.add("stop_workers"); return nil }, + stopNotificationWorkers: func(ctx context.Context) error { + tr.add("stop_notification_workers") + // A final delivery pass against an unreachable Slack: it must + // honour the shutdown deadline, never outlive it. + <-ctx.Done() + return ctx.Err() + }, + } + done := make(chan error, 1) + go func() { done <- seq.run(ctx) }() + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("run err = %v, want the shutdown context's deadline error", err) + } + case <-time.After(5 * time.Second): + t.Fatal("shutdown never returned: a Slack outage held the process open") + } + assertTrace(t, tr.snapshot(), []string{ + "stop_receivers", "stop_correlator", "stop_workers", "stop_notification_workers", + }) +} + // TestFoundationStopSequenceDrainsResultingInputsToQuiescence proves the // drain is a loop: a controller/Triage drain that handles work (and so may // have appended fresh Situation inputs) is followed by another foundation diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go index 9bf35b1..42aa564 100644 --- a/cmd/alertint/situation_notifications.go +++ b/cmd/alertint/situation_notifications.go @@ -6,9 +6,16 @@ import ( "context" "errors" "fmt" + "log/slog" + "os" + "strings" "time" + "github.com/alertint/alertint-agent/internal/audit" + "github.com/alertint/alertint-agent/internal/config" + "github.com/alertint/alertint-agent/internal/notify/slack" + "github.com/alertint/alertint-agent/internal/notify/stdout" "github.com/alertint/alertint-agent/internal/situation" "github.com/alertint/alertint-agent/internal/situation/model" "github.com/alertint/alertint-agent/internal/store" @@ -431,3 +438,358 @@ func classifyDeliveryError(err error) error { } return &deliveryAdapterError{class: situation.DeliveryRetryable, code: "delivery_failed", err: err} } + +// ---------------------------------------------------------------------- +// Situation notification runtime (Plan 3 Task 9) +// +// notificationRuntime bundles the two Plan 3 consumers of already-committed +// history — the Situation notification worker (the single reachable Slack +// writer) and the stdout Transition-stream worker — plus the startup-only +// recovery pass both depend on having already run. +// +// It is a THIRD runtime alongside foundationRuntime and controllerRuntime +// rather than an extension of either, and deliberately so. controllerRuntime +// exposes Drain, because its workers are producers whose queued work +// shutdown must drain to quiescence; these two workers must never be +// drained (R6: an external Slack outage would spin the bounded drain loop to +// its round cap and hold the whole process open). Giving them a Drain they +// may never be called with would make that guarantee a comment; leaving them +// in their own runtime with only Start/Stop makes it structural. The second +// reason is construction: the notification worker exists only when Situation +// Slack is configured, while the stdout stream always runs, and folding an +// optional Slack dependency into the always-present controller runtime would +// leak that optionality into every controller call site. +// +// main constructs exactly one per process and composes it into +// foundationSequence (recoverNotificationWork, startNotificationWorkers) and +// foundationStopSequence (stopNotificationWorkers, last). +// ---------------------------------------------------------------------- + +// notificationStartupStore is exactly the durable surface the startup pass +// drives. *store.Store satisfies it — asserted below. +type notificationStartupStore interface { + RecoverExpiredNotificationClaims(ctx context.Context, now time.Time) (int, error) + RecoverExpiredTransitionStreamClaims(ctx context.Context, now time.Time) (int, error) + ScheduleSituationsMissingFirstTransition(ctx context.Context, now time.Time) (int, error) + ScheduleSituationsWithStaleRootProjection(ctx context.Context, now time.Time) (int, error) + GetSlackDeliveryState(ctx context.Context) (situation.SlackDeliveryState, error) + RecoverDeliveryGap(ctx context.Context, now time.Time) (string, bool, error) +} + +// slackConfigurationProbe is the readiness check startup step 4 runs. +// *SituationDeliverer satisfies it; it is nil when Slack is not configured. +type slackConfigurationProbe interface { + Probe(ctx context.Context) error +} + +// situationNotificationWorker is exactly what this runtime drives on the +// notification worker. *situation.NotificationWorker satisfies it. +type situationNotificationWorker interface { + Start(ctx context.Context) + Stop(ctx context.Context) error + ReactivateConfiguration(ctx context.Context) (int, error) +} + +// transitionStreamWorker is exactly what this runtime drives on the stdout +// stream worker. *stdout.TransitionStreamWorker satisfies it. +type transitionStreamWorker interface { + Start(ctx context.Context) + Stop(ctx context.Context) error +} + +var ( + _ notificationStartupStore = (*store.Store)(nil) + _ stdout.TransitionStreamStore = (*store.Store)(nil) +) + +// notificationRuntime owns the Slack notification worker (nil when +// Situation Slack is not configured) and the stdout Transition-stream +// worker (always present — the state stream is not Slack-gated). +type notificationRuntime struct { + store notificationStartupStore + probe slackConfigurationProbe + worker situationNotificationWorker + stream transitionStreamWorker + logger *slog.Logger +} + +// newNotificationRuntime wires the runtime. worker/probe may both be nil +// (Situation Slack disabled or unconfigured), in which case durable blocked +// intents are retained untouched and only the stdout stream runs. stream may +// be nil only in tests that exercise the startup pass alone. +func newNotificationRuntime(st notificationStartupStore, probe slackConfigurationProbe, + worker situationNotificationWorker, stream transitionStreamWorker, logger *slog.Logger) *notificationRuntime { + if logger == nil { + logger = slog.Default() + } + return ¬ificationRuntime{store: st, probe: probe, worker: worker, stream: stream, logger: logger} +} + +// notificationRecovery is the startup pass's report, for +// logNotificationRecoveryReport's sibling logging call. +type notificationRecovery struct { + NotificationClaimsRecovered int + StreamClaimsRecovered int + ScheduledMissingHistory int + ScheduledStaleRoot int + SlackConfigurationValid bool + ConfigurationGeneration int64 + Reactivated int + BlockedConfigurationRetained int + GapReplayResumed bool + GapGeneration string +} + +// RecoverAndReactivate runs spec.md's own startup steps 2 through 6, in that +// exact order, after Plan 1/2 reconstruction and Plan 2's controller +// recovery and BEFORE any worker or Receiver starts: +// +// 2. recover expired notification and stdout-stream claims; +// 3. schedule every nonterminal Situation missing its first Transition; +// 4. validate the Slack configuration and record its generation; +// 5. reactivate eligible configuration-blocked intents; and +// 6. perform stale-root supersession and resume ordered recovery replay. +// +// It publishes nothing. Steps 3 and 6 only pull a Situation's next +// assessment forward, so the ordinary controller path decides — under the +// ordinary materiality and publication rules — whether anything is committed +// at all; a restart on unchanged truth therefore creates no Transition and +// no Slack effect ("Startup never publishes merely because the binary +// restarted"). +// +// A failed Slack probe is NOT an error: an unreachable or misconfigured +// Slack at boot is an ordinary delay, so steps 5 and 6 are skipped, every +// blocked intent stays durably blocked, and the process still starts. Only a +// genuine Store failure returns an error — and then the caller must not +// start Receivers. +func (r *notificationRuntime) RecoverAndReactivate(ctx context.Context, now time.Time) (notificationRecovery, error) { + var report notificationRecovery + + recovered, err := r.store.RecoverExpiredNotificationClaims(ctx, now) + if err != nil { + return report, fmt.Errorf("situation notifications: recover expired notification claims: %w", err) + } + report.NotificationClaimsRecovered = recovered + + recoveredStream, err := r.store.RecoverExpiredTransitionStreamClaims(ctx, now) + if err != nil { + return report, fmt.Errorf("situation notifications: recover expired transition stream claims: %w", err) + } + report.StreamClaimsRecovered = recoveredStream + + scheduled, err := r.store.ScheduleSituationsMissingFirstTransition(ctx, now) + if err != nil { + return report, fmt.Errorf("situation notifications: schedule situations missing a first transition: %w", err) + } + report.ScheduledMissingHistory = scheduled + + report.SlackConfigurationValid = r.validateSlackConfiguration(ctx) + + state, err := r.store.GetSlackDeliveryState(ctx) + if err != nil { + return report, fmt.Errorf("situation notifications: read slack delivery state: %w", err) + } + report.ConfigurationGeneration = state.ConfigurationGeneration + report.BlockedConfigurationRetained = state.BlockedConfigurationCount + + if !report.SlackConfigurationValid { + return report, nil + } + if err := r.reactivateAndResume(ctx, now, &report); err != nil { + return report, err + } + return report, nil +} + +// validateSlackConfiguration is startup step 4. No probe at all (Slack +// disabled) and a failed probe are both "not validated": neither may +// reactivate a blocked intent, since only a corrected configuration may, and +// neither is a startup failure. +func (r *notificationRuntime) validateSlackConfiguration(ctx context.Context) bool { + if r.probe == nil { + return false + } + if err := r.probe.Probe(ctx); err != nil { + r.logger.Warn("situation notifications: slack configuration did not validate at startup; "+ + "blocked effects are retained and delivery is delayed", slog.String("err", err.Error())) + return false + } + return true +} + +// reactivateAndResume is startup steps 5 and 6, reached only once the Slack +// configuration has actually validated. +func (r *notificationRuntime) reactivateAndResume(ctx context.Context, now time.Time, report *notificationRecovery) error { + if r.worker != nil { + // Exactly once per process (the worker's own one-shot guard makes + // this idempotent against its first steady-state probe, whichever + // runs first). + n, err := r.worker.ReactivateConfiguration(ctx) + if err != nil { + return fmt.Errorf("situation notifications: reactivate configuration-blocked intents: %w", err) + } + report.Reactivated = n + if n > 0 && report.BlockedConfigurationRetained >= n { + report.BlockedConfigurationRetained -= n + } + } + staleRoots, err := r.store.ScheduleSituationsWithStaleRootProjection(ctx, now) + if err != nil { + return fmt.Errorf("situation notifications: schedule situations with a stale root projection: %w", err) + } + report.ScheduledStaleRoot = staleRoots + + generation, resumed, err := r.store.RecoverDeliveryGap(ctx, now) + if err != nil { + return fmt.Errorf("situation notifications: resume delivery gap replay: %w", err) + } + report.GapReplayResumed = resumed + report.GapGeneration = generation + return nil +} + +// Start launches the notification worker, then the stdout stream worker, +// each on its own background schedule. Call only after RecoverAndReactivate +// has succeeded, and after the controller/Triage workers have started. +func (r *notificationRuntime) Start(ctx context.Context) { + if r.worker != nil { + r.worker.Start(ctx) + } + if r.stream != nil { + r.stream.Start(ctx) + } +} + +// Stop stops the stdout stream worker, then the notification worker — the +// reverse of Start, mirroring the other two runtimes' stop-in-reverse +// discipline. Each runs its own ONE bounded final pass under ctx and then +// releases every claim it still holds (R6). Neither ever joins the shutdown +// drain rounds, so an unreachable Slack cannot hold the process open past +// the shutdown context's own deadline; whatever stays pending is reclaimed +// at the next startup. +func (r *notificationRuntime) Stop(ctx context.Context) error { + var errs []error + if r.stream != nil { + if err := r.stream.Stop(ctx); err != nil { + errs = append(errs, fmt.Errorf("transition stream worker stop: %w", err)) + } + } + if r.worker != nil { + if err := r.worker.Stop(ctx); err != nil { + errs = append(errs, fmt.Errorf("notification worker stop: %w", err)) + } + } + return errors.Join(errs...) +} + +// logNotificationRecoveryReport logs one RecoverAndReactivate pass at +// startup — the sibling of logReconstructionReport and +// logControllerRecoveryReport. +func logNotificationRecoveryReport(logger *slog.Logger, report notificationRecovery) { + logger.Info("situation notifications recovered", + slog.Int("notification_claims_recovered", report.NotificationClaimsRecovered), + slog.Int("transition_stream_claims_recovered", report.StreamClaimsRecovered), + slog.Int("situations_scheduled_for_first_transition", report.ScheduledMissingHistory), + slog.Int("situations_scheduled_for_stale_root", report.ScheduledStaleRoot), + slog.Bool("slack_configuration_valid", report.SlackConfigurationValid), + slog.Int64("slack_configuration_generation", report.ConfigurationGeneration), + slog.Int("configuration_blocked_reactivated", report.Reactivated), + slog.Int("configuration_blocked_retained", report.BlockedConfigurationRetained), + slog.Bool("gap_replay_resumed", report.GapReplayResumed), + ) + if report.BlockedConfigurationRetained > 0 { + logger.Warn("situation notifications: durable effects remain blocked on Slack configuration; "+ + "they are retained, never failed, and reactivate after a restart with corrected configuration", + slog.Int("blocked_effects", report.BlockedConfigurationRetained)) + } +} + +// runNotificationRecovery runs one notificationRuntime.RecoverAndReactivate +// pass and logs its report — the recoverNotificationWork step of runServe's +// own startupSeq. A named function, not a closure, for the same +// gocyclo reason runFoundationReconstruction and runControllerRecovery are. +func runNotificationRecovery(ctx context.Context, nrt *notificationRuntime, logger *slog.Logger) error { + report, err := nrt.RecoverAndReactivate(ctx, time.Now().UTC()) + if err != nil { + return fmt.Errorf("situation notification recovery: %w", err) + } + logNotificationRecoveryReport(logger, report) + return nil +} + +// buildSituationNotificationRuntime assembles the process's one +// notificationRuntime from configuration. +// +// The stdout Transition-stream worker is ALWAYS built: spec.md makes +// Situation Transition events the authoritative outward state stream, and a +// silent, floor-withheld, or Slack-disabled installation still emits its +// complete history there. `notify.stdout` gates the legacy Finding JSON +// lines only, never this stream. +// +// The Slack notification worker is built only when Situation Slack is +// actually usable: enabled, with a resolvable bot token and a channel. When +// it is not, no Slack credential is constructed on this path at all and +// every durable blocked intent is retained untouched — never failed, never +// silently dropped — so a restart with corrected configuration reactivates +// it. Together with buildNotifier's System-message-only Slack notifier +// (ADR-0042/ADR-0046), these are the only two places in production that +// receive a Slack credential. +func buildSituationNotificationRuntime(cfg *config.Config, st *store.Store, auditor *audit.Auditor, + owner string, logger *slog.Logger) *notificationRuntime { + // TRUE nil interfaces, never typed nils: a nil *audit.Auditor stored in + // an interface is non-nil and would panic on its first Append (the same + // typed-nil trap sentryErrorSource/zabbixContextSource guard against). + var streamAudit stdout.TransitionStreamAuditSink + var deliveryAudit situation.AuditSink + if auditor != nil { + streamAudit = auditor + deliveryAudit = auditor + } + stream := stdout.NewTransitionStreamWorker(os.Stdout, st, + stdout.TransitionStreamConfig{Owner: owner + ":transition-stream"}, streamAudit, + func() time.Time { return time.Now().UTC() }, logger) + + probe, worker := buildSituationSlackWorker(cfg, st, owner, deliveryAudit, logger) + if worker == nil { + logger.Info("situation notifications: slack delivery is not configured; "+ + "Situation history is complete in the store, MCP, audit, and the stdout Transition stream", + slog.Bool("slack_enabled", cfg.Notify.Slack.Enabled)) + } else { + logger.Info("situation notifications: slack delivery ready", + slog.String("slack_channel", cfg.Notify.Slack.Channel)) + } + return newNotificationRuntime(st, probe, worker, stream, logger) +} + +// buildSituationSlackWorker resolves the Slack credential and, when it is +// usable, constructs the deliverer and the single notification worker. It +// returns (nil, nil) — TRUE nil interfaces, never typed nils — whenever +// Situation Slack cannot be used. +func buildSituationSlackWorker(cfg *config.Config, st *store.Store, owner string, + auditSink situation.AuditSink, logger *slog.Logger) (slackConfigurationProbe, situationNotificationWorker) { + if !cfg.Notify.Slack.Enabled || strings.TrimSpace(cfg.Notify.Slack.Channel) == "" { + return nil, nil + } + token, err := cfg.SlackBotToken() + if err != nil || strings.TrimSpace(token) == "" { + logger.Warn("situation notifications: slack is enabled but no bot token resolved; " + + "Situation Slack delivery stays off and durable effects are retained") + return nil, nil + } + api := slack.NewClient(slack.Config{BotToken: token}) + deliverer := NewSituationDeliverer(st, api, cfg.Notify.Slack.Channel, func() time.Time { return time.Now().UTC() }) + worker := situation.NewNotificationWorker(st, deliverer, + situation.NotificationWorkerConfig{ + // Lease/heartbeat reuse Plan 2's existing situations.* settings + // (plan.md: "Plan 3 adds no duplicate notification knobs"); Poll, + // Batch, the retry schedule, and the five-minute gap threshold are + // the protocol's own constants, not operator knobs. + Owner: owner + ":notifications", + Lease: time.Duration(cfg.Situations.LeaseSeconds) * time.Second, + Heartbeat: time.Duration(cfg.Situations.HeartbeatSeconds) * time.Second, + Poll: time.Duration(cfg.Situations.ReconcilePollSeconds) * time.Second, + }, + func() time.Time { return time.Now().UTC() }, logger) + worker.SetAuditSink(auditSink) + return deliverer, worker +} diff --git a/cmd/alertint/situation_notifications_test.go b/cmd/alertint/situation_notifications_test.go index 4194516..c74b414 100644 --- a/cmd/alertint/situation_notifications_test.go +++ b/cmd/alertint/situation_notifications_test.go @@ -6,7 +6,9 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" + "sync" "testing" "time" @@ -694,3 +696,335 @@ func TestSituationDelivererDeliverRejectsInvalidIntent(t *testing.T) { t.Fatal("an invalid intent must never reach Slack") } } + +// ---------------------------------------------------------------------- +// notificationRuntime: startup recovery, worker lifecycle, shutdown (Task 9) +// ---------------------------------------------------------------------- + +// nrTracer records the exact order the startup pass drives its steps in. +type nrTracer struct { + mu sync.Mutex + trace []string +} + +func (n *nrTracer) add(s string) { + n.mu.Lock() + defer n.mu.Unlock() + n.trace = append(n.trace, s) +} + +func (n *nrTracer) snapshot() []string { + n.mu.Lock() + defer n.mu.Unlock() + out := make([]string, len(n.trace)) + copy(out, n.trace) + return out +} + +type nrFakeStore struct { + tr *nrTracer + state situation.SlackDeliveryState + gapID string + gapDone bool +} + +func (f *nrFakeStore) RecoverExpiredNotificationClaims(context.Context, time.Time) (int, error) { + f.tr.add("recover_notification_claims") + return 2, nil +} + +func (f *nrFakeStore) RecoverExpiredTransitionStreamClaims(context.Context, time.Time) (int, error) { + f.tr.add("recover_transition_stream_claims") + return 1, nil +} + +func (f *nrFakeStore) ScheduleSituationsMissingFirstTransition(context.Context, time.Time) (int, error) { + f.tr.add("schedule_situations_missing_first_transition") + return 3, nil +} + +func (f *nrFakeStore) ScheduleSituationsWithStaleRootProjection(context.Context, time.Time) (int, error) { + f.tr.add("schedule_stale_root_projections") + return 1, nil +} + +func (f *nrFakeStore) GetSlackDeliveryState(context.Context) (situation.SlackDeliveryState, error) { + f.tr.add("read_slack_delivery_state") + return f.state, nil +} + +func (f *nrFakeStore) RecoverDeliveryGap(context.Context, time.Time) (string, bool, error) { + f.tr.add("resume_gap_replay") + return f.gapID, f.gapDone, nil +} + +type nrFakeProbe struct { + tr *nrTracer + err error +} + +func (p *nrFakeProbe) Probe(context.Context) error { + p.tr.add("validate_slack_configuration") + return p.err +} + +type nrFakeWorker struct { + tr *nrTracer + reactivated int + stopBlocks bool + started bool + stopCallCount int +} + +func (w *nrFakeWorker) Start(context.Context) { + w.tr.add("start_notification_worker") + w.started = true +} + +func (w *nrFakeWorker) Stop(ctx context.Context) error { + w.tr.add("stop_notification_worker") + w.stopCallCount++ + if w.stopBlocks { + <-ctx.Done() + return ctx.Err() + } + return nil +} + +func (w *nrFakeWorker) ReactivateConfiguration(context.Context) (int, error) { + w.tr.add("reactivate_configuration_blocked") + w.reactivated++ + return 2, nil +} + +type nrFakeStream struct{ tr *nrTracer } + +func (s *nrFakeStream) Start(context.Context) { s.tr.add("start_transition_stream_worker") } +func (s *nrFakeStream) Stop(context.Context) error { + s.tr.add("stop_transition_stream_worker") + return nil +} + +// nrRuntime takes its probe/worker as INTERFACES so a test can pass a true +// nil (Slack disabled) rather than a typed-nil pointer that would read as a +// present dependency. +func nrRuntime(tr *nrTracer, st *nrFakeStore, probe slackConfigurationProbe, worker situationNotificationWorker) *notificationRuntime { + return ¬ificationRuntime{ + store: st, + probe: probe, + worker: worker, + stream: &nrFakeStream{tr: tr}, + logger: slog.New(slog.DiscardHandler), + } +} + +// TestSituationNotificationRuntimeStartupFollowsTheSpecOrder pins spec.md's +// own startup steps 2-6, in order, between Plan 1/2 reconstruction and the +// workers starting: recover abandoned notification and stream claims; +// schedule every nonterminal Situation missing its first Transition; +// validate the Slack configuration and read back its durable generation; +// reactivate configuration-blocked intents; then schedule stale root +// projections and resume any interrupted gap replay. +func TestSituationNotificationRuntimeStartupFollowsTheSpecOrder(t *testing.T) { + tr := &nrTracer{} + generation := int64(7) + st := &nrFakeStore{tr: tr, state: situation.SlackDeliveryState{ + ConfigurationGeneration: generation, BlockedConfigurationCount: 2, + }, gapID: "gap-1", gapDone: true} + worker := &nrFakeWorker{tr: tr} + rt := nrRuntime(tr, st, &nrFakeProbe{tr: tr}, worker) + + report, err := rt.RecoverAndReactivate(context.Background(), time.Now().UTC()) + if err != nil { + t.Fatalf("RecoverAndReactivate: %v", err) + } + want := []string{ + "recover_notification_claims", + "recover_transition_stream_claims", + "schedule_situations_missing_first_transition", + "validate_slack_configuration", + "read_slack_delivery_state", + "reactivate_configuration_blocked", + "schedule_stale_root_projections", + "resume_gap_replay", + } + got := tr.snapshot() + if len(got) != len(want) { + t.Fatalf("startup trace = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("startup trace = %v, want %v", got, want) + } + } + if report.ConfigurationGeneration != generation { + t.Errorf("configuration generation = %d, want %d", report.ConfigurationGeneration, generation) + } + if report.Reactivated != 2 || report.NotificationClaimsRecovered != 2 || report.StreamClaimsRecovered != 1 { + t.Errorf("report = %+v, want the fakes' own counts", report) + } + if !report.GapReplayResumed { + t.Error("an interrupted gap replay was not reported as resumed") + } + if worker.reactivated != 1 { + t.Errorf("ReactivateConfiguration called %d times, want exactly 1", worker.reactivated) + } +} + +// TestSituationNotificationRuntimeStartupKeepsBlockedIntentsWhenSlackFails +// proves a Slack configuration that does not validate at boot is an ordinary +// delay, never a startup failure and never a reactivation: blocked intents +// stay durably blocked, no gap is recovered, and the process still starts. +func TestSituationNotificationRuntimeStartupKeepsBlockedIntentsWhenSlackFails(t *testing.T) { + tr := &nrTracer{} + st := &nrFakeStore{tr: tr, state: situation.SlackDeliveryState{BlockedConfigurationCount: 4}} + worker := &nrFakeWorker{tr: tr} + rt := nrRuntime(tr, st, &nrFakeProbe{tr: tr, err: errors.New("invalid_auth")}, worker) + + report, err := rt.RecoverAndReactivate(context.Background(), time.Now().UTC()) + if err != nil { + t.Fatalf("RecoverAndReactivate must not fail on an unreachable Slack: %v", err) + } + if report.SlackConfigurationValid { + t.Error("report claims a valid Slack configuration after a failed probe") + } + if report.BlockedConfigurationRetained != 4 { + t.Errorf("retained blocked intents = %d, want 4", report.BlockedConfigurationRetained) + } + if worker.reactivated != 0 { + t.Error("configuration-blocked intents were reactivated on a failed probe") + } + for _, phase := range tr.snapshot() { + if phase == "resume_gap_replay" { + t.Error("a gap was recovered while Slack was still unreachable") + } + } +} + +// TestSituationNotificationRuntimeStartupWithoutSlackRetainsDurableWork +// proves the Slack-disabled assembly: no worker and no probe exist at all, +// the claim-recovery and scheduling steps still run, and every durable +// blocked intent is retained rather than failed. +func TestSituationNotificationRuntimeStartupWithoutSlackRetainsDurableWork(t *testing.T) { + tr := &nrTracer{} + st := &nrFakeStore{tr: tr, state: situation.SlackDeliveryState{BlockedConfigurationCount: 3}} + rt := nrRuntime(tr, st, nil, nil) + + report, err := rt.RecoverAndReactivate(context.Background(), time.Now().UTC()) + if err != nil { + t.Fatalf("RecoverAndReactivate: %v", err) + } + if report.SlackConfigurationValid { + t.Error("report claims a valid Slack configuration with Slack disabled") + } + if report.BlockedConfigurationRetained != 3 { + t.Errorf("retained blocked intents = %d, want 3", report.BlockedConfigurationRetained) + } + want := []string{ + "recover_notification_claims", + "recover_transition_stream_claims", + "schedule_situations_missing_first_transition", + "read_slack_delivery_state", + } + got := tr.snapshot() + if len(got) != len(want) { + t.Fatalf("startup trace = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("startup trace = %v, want %v", got, want) + } + } +} + +// TestSituationNotificationRuntimeStartsAndStopsInMirroredOrder pins the +// lifecycle order: the notification worker starts first and stops last, with +// the stdout stream worker inside it. +func TestSituationNotificationRuntimeStartsAndStopsInMirroredOrder(t *testing.T) { + tr := &nrTracer{} + st := &nrFakeStore{tr: tr} + rt := nrRuntime(tr, st, &nrFakeProbe{tr: tr}, &nrFakeWorker{tr: tr}) + + rt.Start(context.Background()) + if err := rt.Stop(context.Background()); err != nil { + t.Fatalf("Stop: %v", err) + } + want := []string{ + "start_notification_worker", "start_transition_stream_worker", + "stop_transition_stream_worker", "stop_notification_worker", + } + got := tr.snapshot() + if len(got) != len(want) { + t.Fatalf("lifecycle trace = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("lifecycle trace = %v, want %v", got, want) + } + } +} + +// TestSituationNotificationRuntimeStopEndsWithinTheShutdownContext is R6's +// operational guarantee: a final delivery pass wedged on an unreachable +// Slack ends when the shutdown context does. The runtime reports the +// deadline rather than hanging, and its committed intents stay pending for +// the next startup to reclaim — they are never failed. +func TestSituationNotificationRuntimeStopEndsWithinTheShutdownContext(t *testing.T) { + tr := &nrTracer{} + st := &nrFakeStore{tr: tr} + worker := &nrFakeWorker{tr: tr, stopBlocks: true} + rt := nrRuntime(tr, st, &nrFakeProbe{tr: tr}, worker) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { done <- rt.Stop(ctx) }() + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Stop err = %v, want the shutdown context's deadline error", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Stop never returned: a Slack outage held shutdown open") + } + if worker.stopCallCount != 1 { + t.Fatalf("notification worker stopped %d times, want exactly one bounded final pass", worker.stopCallCount) + } +} + +// TestSituationNotificationRuntimeRestartAlonePublishesNothing is the +// spec's "startup never publishes merely because the binary restarted" +// invariant, against a real store: two full startup passes over a Situation +// that has no history yet create no Transition, no Episode summary, and no +// notification intent — they only make it due, so the ordinary controller +// path decides, under the ordinary materiality rules, what (if anything) to +// commit. +func TestSituationNotificationRuntimeRestartAlonePublishesNothing(t *testing.T) { + st := newTestFoundationStore(t) + now := time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) + sitID := seedControllerRuntimeSituation(t, st, "group-restart", now) + + rt := newNotificationRuntime(st, nil, nil, nil, slog.New(slog.DiscardHandler)) + for pass := range 2 { + if _, err := rt.RecoverAndReactivate(context.Background(), now); err != nil { + t.Fatalf("RecoverAndReactivate pass %d: %v", pass+1, err) + } + for _, q := range []struct { + table string + query string + }{ + {"situation_transitions", `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`}, + {"situation_episode_summaries", `SELECT COUNT(*) FROM situation_episode_summaries WHERE situation_id = ?`}, + {"notification_intents", `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ?`}, + {"situation_transition_stream", `SELECT COUNT(*) FROM situation_transition_stream WHERE situation_id = ?`}, + } { + var n int + if err := st.DB().QueryRowContext(context.Background(), q.query, sitID).Scan(&n); err != nil { + t.Fatalf("count %s: %v", q.table, err) + } + if n != 0 { + t.Fatalf("pass %d created %d %s rows; a restart alone must publish nothing", pass+1, n, q.table) + } + } + } +} diff --git a/docs/integrations/mcp-clients.md b/docs/integrations/mcp-clients.md index 32d3f91..df141c6 100644 --- a/docs/integrations/mcp-clients.md +++ b/docs/integrations/mcp-clients.md @@ -136,7 +136,9 @@ restart Windsurf and check **Settings → MCP Servers**: | `alertint_get_evidence_pack` | Get the evidence pack and Prometheus metrics for an incident. | | `alertint_verify_audit` | Verify the hash-chained audit log and report any tampering. | | `alertint_list_situations` | List durable Situations — the exact-group lineage that durably owns one or more Incidents — most recently updated first. A bounded summary: lifecycle/attention/scheduling fields and due reasons only, no Assessment or controller detail (use `alertint_get_situation` for that) and no Slack presence. | -| `alertint_get_situation` | Get one Situation by id or public handle: its immutable member Incidents (each with its current Triage decision/phase/attempts/due time/covered digests), the current authoritative Assessment and derivation, current operator contract, material/Assessment-basis hashes, the eligible Sufficient-reason candidate set (`eligible_reasons`: identity, code, catalog/predicate versions, evidence references, deterministic-floor flag), up to 20 bounded sanitized recent Assessment attempts, and controller retry/park state. `assessment`/`operator_contract`/the hash fields render as explicit `null`, and `recent_attempts` and `eligible_reasons` as empty arrays, for a Situation the controller has not reconciled at least once yet — never an error, and never a fabricated placeholder. | +| `alertint_get_situation` | Get one Situation by id or public handle: its immutable member Incidents (each with its current Triage decision/phase/attempts/due time/covered digests), the current authoritative Assessment and derivation, current operator contract, material/Assessment-basis hashes, the eligible Sufficient-reason candidate set (`eligible_reasons`: identity, code, catalog/predicate versions, evidence references, deterministic-floor flag), up to 20 bounded sanitized recent Assessment attempts, and controller retry/park state. `assessment`/`operator_contract`/the hash fields render as explicit `null`, and `recent_attempts` and `eligible_reasons` as empty arrays, for a Situation the controller has not reconciled at least once yet — never an error, and never a fabricated placeholder. Also carries `episode` — the current Episode summary read coherently with the exact Transition it was folded from (explicit `null` before any Transition exists) — and `slack_delivery`, the Situation's whole Slack presence: whether a root is durably published, where it lives, and every durable delivery obligation's class, status, priority, retry/supersession reason, and delivered coordinates. | +| `alertint_list_situation_transitions` | Page one Situation's immutable Transition journal, oldest first. Each Transition is one authoritative material change: lifecycle/attention, operator contract, transition reason, journal kind and bounded journal entry, evidence references, actor, and drill marker. History is never reconstructed from current state — a Situation with no Transition yet returns an empty array. Page with the returned `next_cursor`, a stable `(sequence, id)` position rather than an offset. | +| `alertint_get_delivery_state` | Get the installation-level Situation Slack delivery state: the continuous-failure window, the durable Slack configuration generation and how many effects are blocked on it, the current Delivery-gap generation with its status/age and replay backlog, retries and outcomes by effect class, how many outcomes Slack never confirmed either way, and the stdout Transition-stream backlog. Bounded counts and closed codes only — never a token, a Slack response, or a provider error body. | | `prometheus_query` | Instant PromQL query against the connected Prometheus (requires Prometheus enabled). | | `prometheus_query_range` | Range PromQL query with auto-stepped resolution (requires Prometheus enabled). | | `loki_query_range` | Range-query the configured log backend using its native query language (requires a log source enabled). | diff --git a/internal/audit/audit.go b/internal/audit/audit.go index 35c8db6..559fdc6 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -217,3 +217,129 @@ func validateAppendArgs(actor, kind string) error { } return nil } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the Situation history and delivery event catalog. +// +// Audit event kinds are ordinary strings elsewhere in this codebase, written +// at their emitting call site. Plan 3's are named here instead for one +// reason: spec.md requires a specific, complete set of events, and plan.md +// requires that none of them collide with Plan 2's existing catalog. A +// catalog stated in one place is the only way a test can prove both — and +// this package, which every emitter already depends on, is the one place +// that cannot create an import cycle. +// +// Every Plan 3 kind lives under exactly one of three dotted prefixes: +// situation.history.* (what durably happened to a Situation's operator +// history), situation.notification.* (what happened to a durable Slack +// delivery obligation), and situation.transition_stream.* (what happened to +// the stdout state stream). Plan 2's names are underscore-separated after +// "situation." (situation.assessment_*, situation.triage_*) or a different +// dotted namespace (situation.controller.commit_failed, incident.triage_*), +// so the two families cannot overlap. +// ---------------------------------------------------------------------- + +const ( + // KindHistoryTransitionCommitted records one immutable Transition + // landing in the fenced controller commit. + KindHistoryTransitionCommitted = "situation.history.transition_committed" + // KindHistorySummaryProjected records the Episode-summary version that + // commit folded. + KindHistorySummaryProjected = "situation.history.summary_projected" + // KindHistoryArtifactJournaled records one operator artifact consumed + // into an operator_artifact_recorded Transition (R1). + KindHistoryArtifactJournaled = "situation.history.artifact_journaled" + // KindHistoryArtifactOwnerTerminal records an artifact that reached an + // already-terminal owner: visible, never journaled (R2). Its emitter is + // the input-application path (Store.ApplySituationInput, which marks the + // row owner_terminal), which carries no audit seam in this slice — + // ApplySituationInput returns only an error, so the applying worker + // cannot tell that outcome from an ordinary attach. The name is reserved + // here because spec.md's event list requires it and because reserving it + // keeps the collision check honest; wiring the emitter needs + // ApplySituationInput to report the outcome, which is Plan 3's + // input-application contract, not this task's. + KindHistoryArtifactOwnerTerminal = "situation.history.artifact_owner_terminal" + + // KindNotificationIntentCreated records one durable delivery obligation + // created by the authoritative commit. + KindNotificationIntentCreated = "situation.notification.intent_created" + // KindNotificationClaimed records one fenced claim of that obligation. + KindNotificationClaimed = "situation.notification.claimed" + // KindNotificationDelivered records an acknowledged Slack delivery and + // its coordinates. + KindNotificationDelivered = "situation.notification.delivered" + // KindNotificationRetried records one scheduled indefinite retry. + KindNotificationRetried = "situation.notification.retried" + // KindNotificationConfigurationBlocked records a definite token, scope, + // or channel rejection holding the effect until configuration changes. + KindNotificationConfigurationBlocked = "situation.notification.configuration_blocked" + // KindNotificationFailed records the one permanent outcome: a durable + // intent this build proved invalid. It stays operator-redriveable. + KindNotificationFailed = "situation.notification.failed" + // KindNotificationWithheld records a poke the operator's Slack floor + // withheld — a durable decision, never an absent row. + KindNotificationWithheld = "situation.notification.withheld" + // KindNotificationSuperseded records an older root projection retired by + // a newer one. + KindNotificationSuperseded = "situation.notification.superseded" + // KindNotificationGapOpened, ...Recovered, and ...Completed record the + // ADR-0042 Delivery-gap generation lifecycle. + KindNotificationGapOpened = "situation.notification.gap_opened" + KindNotificationGapRecovered = "situation.notification.gap_recovered" + KindNotificationGapCompleted = "situation.notification.gap_completed" + + // KindTransitionStreamEmitted records one acknowledged stdout line; + // KindTransitionStreamFailed records a durable stream row this build + // cannot render at all. + KindTransitionStreamEmitted = "situation.transition_stream.emitted" + KindTransitionStreamFailed = "situation.transition_stream.failed" +) + +// SituationHistoryKinds returns every Plan 3 event kind, in catalog order. +// Its completeness against spec.md's own required event list, and its +// disjointness from ReservedPlan2Kinds, are both pinned by this package's +// tests. +func SituationHistoryKinds() []string { + return []string{ + KindHistoryTransitionCommitted, + KindHistorySummaryProjected, + KindHistoryArtifactJournaled, + KindHistoryArtifactOwnerTerminal, + KindNotificationIntentCreated, + KindNotificationClaimed, + KindNotificationDelivered, + KindNotificationRetried, + KindNotificationConfigurationBlocked, + KindNotificationFailed, + KindNotificationWithheld, + KindNotificationSuperseded, + KindNotificationGapOpened, + KindNotificationGapRecovered, + KindNotificationGapCompleted, + KindTransitionStreamEmitted, + KindTransitionStreamFailed, + } +} + +// ReservedPlan2Kinds is the existing Situation/Incident audit vocabulary +// Plan 3 must not collide with: the controller's Assessment and Triage +// events (internal/situation/controller.go), its commit-failure diagnostic, +// the Triage worker's own skip event, and the Incident Triage exhaustion +// event the startup horizon and the Triage worker both emit. It exists so +// the collision rule is checkable rather than merely stated. +func ReservedPlan2Kinds() []string { + return []string{ + "situation.assessment_authoritative", + "situation.assessment_fallback", + "situation.assessment_reused", + "situation.assessment_stale", + "situation.assessment_call_dispatched", + "situation.assessment_rejected", + "situation.assessment_failed", + "situation.triage_requested", + "situation.triage_skipped", + "situation.controller.commit_failed", + "incident.triage_exhausted", + } +} diff --git a/internal/audit/audit_test.go b/internal/audit/audit_test.go index 159c8de..9793d7b 100644 --- a/internal/audit/audit_test.go +++ b/internal/audit/audit_test.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "testing" "time" @@ -256,3 +257,108 @@ func TestAppend_RollbackLeavesNoRow(t *testing.T) { // Channel referenced to silence "declared and not used" suspicion. _ = fmt.Sprintf("%v", err) } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the Situation history/delivery event catalog +// ---------------------------------------------------------------------- + +// TestSituationAuditCatalogCoversEveryPlan3Event pins that the catalog names +// every event spec.md requires Plan 3 to audit: Transition commit, summary +// projection, intent creation, claim, retry, configuration block, permanent +// failure, delivery, withholding, supersession, operator-artifact linkage +// (including owner_terminal), and gap open/recovery/completion. +func TestSituationAuditCatalogCoversEveryPlan3Event(t *testing.T) { + want := []string{ + "situation.history.transition_committed", + "situation.history.summary_projected", + "situation.history.artifact_journaled", + "situation.history.artifact_owner_terminal", + "situation.notification.intent_created", + "situation.notification.claimed", + "situation.notification.delivered", + "situation.notification.retried", + "situation.notification.configuration_blocked", + "situation.notification.failed", + "situation.notification.withheld", + "situation.notification.superseded", + "situation.notification.gap_opened", + "situation.notification.gap_recovered", + "situation.notification.gap_completed", + "situation.transition_stream.emitted", + "situation.transition_stream.failed", + } + got := map[string]bool{} + for _, kind := range SituationHistoryKinds() { + if got[kind] { + t.Errorf("catalog lists %q twice", kind) + } + got[kind] = true + } + for _, kind := range want { + if !got[kind] { + t.Errorf("catalog is missing the required event %q", kind) + } + } + if len(got) != len(want) { + t.Errorf("catalog has %d kinds, want exactly the %d spec.md names", len(got), len(want)) + } +} + +// TestSituationAuditCatalogNeverCollidesWithPlan2 proves the naming rule +// plan.md Task 9 Step 7 states: every new event lives under +// situation.history.*, situation.notification.*, or +// situation.transition_stream.*, and none of them collides with Plan 2's +// existing situation.assessment_*, situation.triage_*, +// situation.controller.commit_failed, or incident.triage_* catalog. +func TestSituationAuditCatalogNeverCollidesWithPlan2(t *testing.T) { + reserved := map[string]bool{} + for _, kind := range ReservedPlan2Kinds() { + reserved[kind] = true + } + if len(reserved) == 0 { + t.Fatal("the reserved Plan 2 catalog is empty; the collision check would prove nothing") + } + prefixes := []string{"situation.history.", "situation.notification.", "situation.transition_stream."} + for _, kind := range SituationHistoryKinds() { + if reserved[kind] { + t.Errorf("new event %q collides with Plan 2's existing catalog", kind) + } + ok := false + for _, p := range prefixes { + if strings.HasPrefix(kind, p) { + ok = true + } + } + if !ok { + t.Errorf("new event %q is outside the three sanctioned Plan 3 prefixes %v", kind, prefixes) + } + // A Plan 2 prefix ("situation.assessment_", "situation.triage_") is + // underscore-separated where Plan 3's are dot-separated, so a + // prefix clash is impossible by construction — assert it anyway, so + // a future rename cannot quietly reintroduce one. + for _, r := range ReservedPlan2Kinds() { + if strings.HasPrefix(kind, r) || strings.HasPrefix(r, kind) { + t.Errorf("new event %q shares a prefix with Plan 2's %q", kind, r) + } + } + } +} + +// TestSituationAuditCatalogAppendsAndVerifies proves every catalog name is +// actually appendable and keeps the hash chain intact — a name the Auditor +// rejects would be a silently missing audit trail. +func TestSituationAuditCatalogAppendsAndVerifies(t *testing.T) { + a, _, ctx := newAuditor(t) + for _, kind := range SituationHistoryKinds() { + if err := a.Append(ctx, "situation.controller", kind, map[string]any{"situation_id": "sit-1"}); err != nil { + t.Fatalf("append %q: %v", kind, err) + } + } + report, err := a.Verify(ctx) + if err != nil || !report.OK { + t.Fatalf("verify after the catalog appends: %v %+v", err, report) + } + if report.RowsChecked != len(SituationHistoryKinds()) { + t.Fatalf("rows = %d, want %d", report.RowsChecked, len(SituationHistoryKinds())) + } +} diff --git a/internal/mcp/docs_drift_test.go b/internal/mcp/docs_drift_test.go index 3250b95..7c62ffb 100644 --- a/internal/mcp/docs_drift_test.go +++ b/internal/mcp/docs_drift_test.go @@ -71,6 +71,10 @@ func TestDriftGate_ToolsDocumented(t *testing.T) { addTool(t14.Name) t15, _ := s.toolGetSituation() addTool(t15.Name) + t16, _ := s.toolListSituationTransitions() + addTool(t16.Name) + t17, _ := s.toolGetDeliveryState() + addTool(t17.Name) documented := documentedToolNames(t) diff --git a/internal/mcp/server.go b/internal/mcp/server.go index b795e76..1ea52d3 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -112,6 +112,12 @@ func NewServer(cfg Config, st *store.Store, auditor *audit.Auditor) *Server { // envelope, Assessment, or reassessment request. ms.AddTool(s.toolListSituations()) ms.AddTool(s.toolGetSituation()) + // Plan 3 Task 9: the two bounded read-only history/delivery surfaces. + // Always registered alongside the two above — there is no connector to + // gate them on, and a Situation with no history yet answers honestly + // rather than erroring. + ms.AddTool(s.toolListSituationTransitions()) + ms.AddTool(s.toolGetDeliveryState()) // Log passthrough tool, registered only when a log source is configured. // Named after the active backend (loki_query_range) so multiple sources can diff --git a/internal/mcp/server_situations.go b/internal/mcp/server_situations.go index 4298e4d..db7b817 100644 --- a/internal/mcp/server_situations.go +++ b/internal/mcp/server_situations.go @@ -201,29 +201,9 @@ func controllerStateRowFrom(r store.ControllerRetryState) controllerStateRow { // a fixed, generic message — never a wrapped store/SQL error — so a lookup // failure can never leak driver or query text to an MCP client. func (s *Server) handleGetSituation(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { - id := mcplib.ParseString(req, "id", "") - handle := mcplib.ParseString(req, "handle", "") - if (id == "") == (handle == "") { - return errResult("exactly one of id or handle is required"), nil - } - - var ( - sit situationmodel.Situation - err error - ) - if id != "" { - sit, err = s.st.GetSituation(ctx, id) - } else { - sit, err = s.st.GetSituationByHandle(ctx, handle) - } - if err != nil { - if errors.Is(err, store.ErrNotFound) { - if id != "" { - return errResult(fmt.Sprintf("situation %q not found", id)), nil - } - return errResult(fmt.Sprintf("situation with handle %q not found", handle)), nil - } - return errResult("failed to get situation"), nil + sit, failed := s.resolveSituation(ctx, req) + if failed != nil { + return failed, nil } members, err := s.st.ListSituationIncidents(ctx, sit.ID) @@ -302,9 +282,360 @@ func (s *Server) handleGetSituation(ctx context.Context, req mcplib.CallToolRequ "controller_state": controllerStateRowFrom(view.Retry), } + // Plan 3 Task 9: the durable operator history and Slack presence. episode + // is an explicit null for a Situation with no Transition yet — history is + // never reconstructed from current state — and slack_delivery always + // answers, saying "published: false" with no effects for a Situation that + // warranted no Slack at all. + episode, delivery, err := s.situationHistoryFor(ctx, sit.ID) + if err != nil { + return errResult("failed to get situation history"), nil + } + if episode == nil { + payload["episode"] = nil + } else { + payload["episode"] = episode + } + payload["slack_delivery"] = delivery + result, err := mcplib.NewToolResultJSON(payload) if err != nil { return errResult("failed to serialize situation: " + err.Error()), nil } return result, nil } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: bounded read-only history and delivery views. +// +// Three additions, all read-only and all reading ONLY Task 5's coherent +// Store views (internal/store/situation_views.go) — never a raw ledger +// join, never a reconstruction of history from current state: +// +// - alertint_get_situation gains "episode" (the current Episode summary +// read in one snapshot with the exact Transition it was folded from) and +// "slack_delivery" (the current root coordinates plus every durable +// effect's status, including the withheld/superseded/delayed decisions +// that are durable rows rather than absent ones); +// - alertint_list_situation_transitions pages the immutable Transition +// journal by a stable (sequence, id) cursor; and +// - alertint_get_delivery_state exposes the installation-level Slack +// delivery snapshot: gap generation/status/age, replay backlog, retries +// by effect class, uncertain outcomes, and the blocked backlog. +// +// None of them returns a claim owner, claim token, lease, bot token, raw +// Slack response, provider error body, or SQL text. The last of those is the +// reason every failure path returns a fixed generic message. +// ---------------------------------------------------------------------- + +func (s *Server) toolListSituationTransitions() (mcplib.Tool, mcpserver.ToolHandlerFunc) { + tool := mcplib.NewTool("alertint_list_situation_transitions", + mcplib.WithDescription("Page one Situation's immutable Transition journal, oldest first. A Transition "+ + "is one authoritative material change: its lifecycle/attention, operator contract, transition reason, "+ + "journal kind and bounded journal entry, evidence references, actor, and drill marker. History is never "+ + "reconstructed from current state — a Situation with no Transition yet returns an empty array. Page with "+ + "the returned next_cursor; it is stable across concurrent commits because it names a (sequence, id) "+ + "position, not an offset."), + mcplib.WithString("id", mcplib.Description("Situation ID. Exactly one of id/handle is required.")), + mcplib.WithString("handle", mcplib.Description("Situation public handle. Exactly one of id/handle is required.")), + mcplib.WithInteger("limit", mcplib.Description("Maximum transitions to return (1-100, default 50).")), + mcplib.WithInteger("cursor_sequence", mcplib.Description("Resume strictly after this Transition sequence (from next_cursor).")), + mcplib.WithString("cursor_id", mcplib.Description("Resume strictly after this Transition id at cursor_sequence (from next_cursor).")), + ) + return tool, s.handleListSituationTransitions +} + +func (s *Server) toolGetDeliveryState() (mcplib.Tool, mcpserver.ToolHandlerFunc) { + tool := mcplib.NewTool("alertint_get_delivery_state", + mcplib.WithDescription("Get the installation-level Situation Slack delivery state: the continuous-failure "+ + "window, the durable Slack configuration generation and how many effects are blocked on it, the current "+ + "Delivery-gap generation with its status/age and replay backlog, retries and outcomes by effect class, "+ + "how many outcomes Slack never confirmed either way, and the stdout Transition-stream backlog. Counts "+ + "and closed codes only — never a token, a Slack response, or a provider error body."), + ) + return tool, s.handleGetDeliveryState +} + +// situationTransitionRow is one immutable Transition as MCP renders it: +// identity, closed codes, hashes, the bounded journal entry, and instants. +type situationTransitionRow struct { + ID string `json:"id"` + Sequence int `json:"sequence"` + InputVersion int `json:"input_version"` + Lifecycle string `json:"lifecycle"` + Attention string `json:"attention"` + Reason string `json:"reason"` + JournalKind string `json:"journal_kind"` + Journal situationmodel.JournalData `json:"journal"` + ActionContract situationmodel.ActionContract `json:"action_contract"` + MaterialFactHash string `json:"material_fact_hash"` + AssessmentID *string `json:"assessment_id"` + SufficientReasonID *string `json:"sufficient_reason_id"` + InterruptionPriority *string `json:"interruption_priority"` + EvidenceRefs []string `json:"evidence_refs"` + Actor string `json:"actor"` + Drill bool `json:"drill"` + CreatedAt time.Time `json:"created_at"` +} + +func situationTransitionRowFrom(t situationmodel.Transition) situationTransitionRow { + row := situationTransitionRow{ + ID: t.ID, Sequence: t.Sequence, InputVersion: t.InputVersion, + Lifecycle: string(t.Lifecycle), Attention: string(t.Attention), + Reason: string(t.Reason), JournalKind: string(t.JournalKind), Journal: t.Journal, + ActionContract: t.ActionContract, MaterialFactHash: t.MaterialFactHash, + AssessmentID: t.AssessmentID, SufficientReasonID: t.SufficientReasonID, + EvidenceRefs: t.EvidenceRefs, Actor: string(t.Actor), Drill: t.Drill, CreatedAt: t.CreatedAt, + } + if row.EvidenceRefs == nil { + row.EvidenceRefs = []string{} + } + if t.InterruptionPriority != nil { + p := string(*t.InterruptionPriority) + row.InterruptionPriority = &p + } + return row +} + +// situationEpisodeRow is the current Episode-summary projection together +// with the exact Transition it was folded from. The two ALWAYS travel +// together: showing a summary beside a source Transition a caller cannot +// see is exactly the incoherence Task 5's snapshot read exists to prevent. +type situationEpisodeRow struct { + Version int `json:"version"` + SourceTransitionSequence int `json:"source_transition_sequence"` + PublicHandle string `json:"public_handle,omitempty"` + Title string `json:"title"` + InitialPublicationReason string `json:"initial_publication_reason,omitempty"` + LatestMaterialReason string `json:"latest_material_reason,omitempty"` + EvidenceConclusion string `json:"evidence_conclusion,omitempty"` + ImpactSummary string `json:"impact_summary,omitempty"` + InvestigationWork []string `json:"investigation_work"` + InvestigationStarted bool `json:"investigation_started"` + CurrentAttention string `json:"current_attention"` + PeakAttention string `json:"peak_attention"` + RecordedOperatorContext []string `json:"recorded_operator_context"` + EffectiveStartedAt time.Time `json:"effective_started_at"` + RecoveryObservedAt *time.Time `json:"recovery_observed_at"` + TerminalAt *time.Time `json:"terminal_at"` + DurationSeconds *int64 `json:"duration_seconds"` + RecurrenceCount int `json:"recurrence_count"` + FinalOutcome string `json:"final_outcome,omitempty"` + RemainingUncertainty string `json:"remaining_uncertainty,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + SourceTransition situationTransitionRow `json:"source_transition"` +} + +func situationEpisodeRowFrom(view store.SituationEpisodeView) situationEpisodeRow { + s := view.Summary + row := situationEpisodeRow{ + Version: s.Version, SourceTransitionSequence: s.SourceTransitionSequence, + PublicHandle: s.PublicHandle, Title: s.Title, + InitialPublicationReason: s.InitialPublicationReason, LatestMaterialReason: s.LatestMaterialReason, + EvidenceConclusion: s.EvidenceConclusion, ImpactSummary: s.ImpactSummary, + InvestigationWork: s.InvestigationWork, InvestigationStarted: s.InvestigationStarted, + CurrentAttention: string(s.CurrentAttention), PeakAttention: string(s.PeakAttention), + RecordedOperatorContext: s.RecordedOperatorContext, EffectiveStartedAt: s.EffectiveStartedAt, + RecoveryObservedAt: s.RecoveryObservedAt, TerminalAt: s.TerminalAt, DurationSeconds: s.DurationSeconds, + RecurrenceCount: s.RecurrenceCount, FinalOutcome: s.FinalOutcome, + RemainingUncertainty: s.RemainingUncertainty, UpdatedAt: s.UpdatedAt, + SourceTransition: situationTransitionRowFrom(view.SourceTransition), + } + if row.InvestigationWork == nil { + row.InvestigationWork = []string{} + } + if row.RecordedOperatorContext == nil { + row.RecordedOperatorContext = []string{} + } + return row +} + +// situationEffectRow is one durable notification intent as MCP renders it. +// Deliberately absent: claim_owner, claim_token, lease_expires_at, and the +// idempotency/client message identities — a delivery obligation's operator +// meaning is its class, subject, status, priority, reason, and where it +// actually landed, never who currently holds its lease. +type situationEffectRow struct { + EffectClass string `json:"effect_class"` + Status string `json:"status"` + TransitionID *string `json:"transition_id"` + TransitionSequence *int `json:"transition_sequence"` + SummaryVersion *int `json:"summary_version"` + MainChannelPoke bool `json:"main_channel_poke"` + InterruptionPriority *string `json:"interruption_priority"` + RequiresRoot bool `json:"requires_root"` + ContractDeadlineAt *time.Time `json:"contract_deadline_at"` + AttemptCount int `json:"attempt_count"` + LastErrorClass *string `json:"last_error_class"` + RetryAt *time.Time `json:"retry_at"` + SupersessionReason *string `json:"supersession_reason"` + DeliveredAs *string `json:"delivered_as"` + Channel *string `json:"channel"` + MessageTS *string `json:"message_ts"` + CreatedAt time.Time `json:"created_at"` + DeliveredAt *time.Time `json:"delivered_at"` +} + +func situationEffectRowFrom(n situationmodel.NotificationIntent) situationEffectRow { + row := situationEffectRow{ + EffectClass: string(n.EffectClass), Status: string(n.Status), + TransitionID: n.TransitionID, TransitionSequence: n.TransitionSequence, + SummaryVersion: n.SummaryVersion, MainChannelPoke: n.MainChannelPoke, + RequiresRoot: n.RequiresRoot, ContractDeadlineAt: n.ContractDeadlineAt, + AttemptCount: n.AttemptCount, LastErrorClass: n.LastErrorClass, RetryAt: n.RetryAt, + SupersessionReason: n.SupersessionReason, DeliveredAs: n.DeliveredAs, + Channel: n.Channel, MessageTS: n.MessageTS, CreatedAt: n.CreatedAt, DeliveredAt: n.DeliveredAt, + } + if n.InterruptionPriority != nil { + p := string(*n.InterruptionPriority) + row.InterruptionPriority = &p + } + return row +} + +// situationDeliveryRow is one Situation's whole Slack presence: whether a +// root is durably published, where it lives, and every durable effect. +type situationDeliveryRow struct { + Published bool `json:"published"` + Channel string `json:"channel,omitempty"` + RootMessageTS string `json:"root_message_ts,omitempty"` + Effects []situationEffectRow `json:"effects"` +} + +// situationHistoryFor reads one Situation's Episode view and Slack delivery +// state through Task 5's bounded readers. A Situation with no Transition yet +// legitimately has no episode at all: that renders as an explicit null, never +// as history reconstructed from current state. +func (s *Server) situationHistoryFor(ctx context.Context, situationID string) (*situationEpisodeRow, situationDeliveryRow, error) { + delivery := situationDeliveryRow{Effects: []situationEffectRow{}} + + channel, messageTS, published, err := s.st.GetSituationRootCoordinates(ctx, situationID) + if err != nil { + return nil, delivery, err + } + delivery.Published = published + if published { + delivery.Channel = channel + delivery.RootMessageTS = messageTS + } + intents, err := s.st.ListSituationNotificationIntents(ctx, situationID, 0) + if err != nil { + return nil, delivery, err + } + for _, intent := range intents { + delivery.Effects = append(delivery.Effects, situationEffectRowFrom(intent)) + } + + view, err := s.st.GetSituationEpisodeView(ctx, situationID) + if errors.Is(err, store.ErrNotFound) { + return nil, delivery, nil + } + if err != nil { + return nil, delivery, err + } + episode := situationEpisodeRowFrom(view) + return &episode, delivery, nil +} + +// transitionCursorRow is the stable page position a caller resumes from. +type transitionCursorRow struct { + Sequence int `json:"sequence"` + ID string `json:"id"` +} + +func (s *Server) handleListSituationTransitions(ctx context.Context, req mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + sit, failed := s.resolveSituation(ctx, req) + if failed != nil { + return failed, nil + } + limit := mcplib.ParseInt(req, "limit", 50) + if limit < 1 { + limit = 50 + } + cursor := store.TransitionCursor{ + Sequence: mcplib.ParseInt(req, "cursor_sequence", 0), + ID: mcplib.ParseString(req, "cursor_id", ""), + } + transitions, err := s.st.ListSituationTransitions(ctx, sit.ID, cursor, limit) + if err != nil { + return errResult("failed to list situation transitions"), nil + } + rows := make([]situationTransitionRow, 0, len(transitions)) + for _, t := range transitions { + rows = append(rows, situationTransitionRowFrom(t)) + } + payload := map[string]any{"situation_id": sit.ID, "transitions": rows, "next_cursor": nil} + // A full page is the only reason to hand back a cursor: a short page has + // nothing after it, and claiming otherwise would make a caller poll + // forever. + if len(rows) == limit && len(rows) > 0 { + last := rows[len(rows)-1] + payload["next_cursor"] = transitionCursorRow{Sequence: last.Sequence, ID: last.ID} + } + result, err := mcplib.NewToolResultJSON(payload) + if err != nil { + return errResult("failed to serialize situation transitions: " + err.Error()), nil + } + return result, nil +} + +func (s *Server) handleGetDeliveryState(ctx context.Context, _ mcplib.CallToolRequest) (*mcplib.CallToolResult, error) { + now := time.Now().UTC() + state, err := s.st.GetSlackDeliveryState(ctx) + if err != nil { + return errResult("failed to get slack delivery state"), nil + } + stats, err := s.st.GetNotificationDeliveryStats(ctx, now) + if err != nil { + return errResult("failed to get notification delivery stats"), nil + } + payload := map[string]any{ + "slack": map[string]any{ + // first_failure_at anchors the CONTINUOUS failure window a gap + // opens after five minutes of; a nil value means Slack delivery + // is currently healthy. + "first_failure_at": state.FirstFailureAt, + "last_success_at": state.LastSuccessAt, + "open_gap_generation": state.OpenGapGeneration, + "open_gap_status": state.OpenGapStatus, + "configuration_generation": state.ConfigurationGeneration, + "blocked_configuration_count": state.BlockedConfigurationCount, + "updated_at": state.UpdatedAt, + }, + "delivery": stats, + } + result, err := mcplib.NewToolResultJSON(payload) + if err != nil { + return errResult("failed to serialize delivery state: " + err.Error()), nil + } + return result, nil +} + +// resolveSituation resolves exactly one of id/handle, returning a ready +// error result rather than an error when the request is unusable. +func (s *Server) resolveSituation(ctx context.Context, req mcplib.CallToolRequest) (situationmodel.Situation, *mcplib.CallToolResult) { + id := mcplib.ParseString(req, "id", "") + handle := mcplib.ParseString(req, "handle", "") + if (id == "") == (handle == "") { + return situationmodel.Situation{}, errResult("exactly one of id or handle is required") + } + var ( + sit situationmodel.Situation + err error + ) + if id != "" { + sit, err = s.st.GetSituation(ctx, id) + } else { + sit, err = s.st.GetSituationByHandle(ctx, handle) + } + if err != nil { + if errors.Is(err, store.ErrNotFound) { + if id != "" { + return situationmodel.Situation{}, errResult(fmt.Sprintf("situation %q not found", id)) + } + return situationmodel.Situation{}, errResult(fmt.Sprintf("situation with handle %q not found", handle)) + } + return situationmodel.Situation{}, errResult("failed to get situation") + } + return sit, nil +} diff --git a/internal/mcp/server_situations_test.go b/internal/mcp/server_situations_test.go index 3f8b426..2b37aa5 100644 --- a/internal/mcp/server_situations_test.go +++ b/internal/mcp/server_situations_test.go @@ -5,6 +5,7 @@ package mcp import ( "context" "encoding/json" + "fmt" "strings" "testing" "time" @@ -170,6 +171,10 @@ func TestGetSituationByIDExactContract(t *testing.T) { // hashes, bounded recent attempts, and controller retry/park state. "assessment_derivation", "material_fact_hash", "assessment_basis_hash", "eligible_reasons", "recent_attempts", "controller_state", + // Plan 3 Task 9 additions: the current Episode summary read + // coherently with its source Transition (explicit null before any + // Transition exists) and the Situation's whole Slack presence. + "episode", "slack_delivery", } if len(payload) != len(wantKeys) { t.Fatalf("payload has %d keys, want exactly %d: %+v", len(payload), len(wantKeys), payload) @@ -479,3 +484,339 @@ func TestGetSituationUnknownDoesNotLeakSQL(t *testing.T) { } } } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: bounded read-only history and delivery views +// ---------------------------------------------------------------------- + +// seedSituationHistoryForMCP writes one Situation's durable Plan 3 history +// straight into the immutable tables the fenced controller commit owns — +// transitions, the current Episode summary, the stdout stream rows, and the +// notification intents they warranted. Raw SQL on purpose: these views are +// read-only, and driving a full ControllerCommit here would test the +// controller, not the view. It returns the Transition IDs it wrote. +func seedSituationHistoryForMCP(t *testing.T, st *store.Store, situationID string, at time.Time, count int) []string { + t.Helper() + ctx := context.Background() + ids := make([]string, 0, count) + for seq := 1; seq <= count; seq++ { + id := fmt.Sprintf("tr-%s-%d", situationID[:8], seq) + ids = append(ids, id) + created := at.Add(time.Duration(seq) * time.Minute).UTC().Format(time.RFC3339Nano) + contract := `{"next_actor":"alertint","alertint_action":"run_acute_triage","alertint_status":"running","next_update_on":["triage_outcome"]}` + journal := fmt.Sprintf(`{"headline":"Material change %d","detail":"bounded journal detail","occurred_at":%q}`, seq, created) + projection := fmt.Sprintf(`{"effective_started_at":%q,"effective_started_at_basis":"source_payload"}`, + at.UTC().Format(time.RFC3339Nano)) + if _, err := st.DB().ExecContext(ctx, ` + INSERT INTO situation_transitions ( + id, situation_id, sequence, input_version, material_fact_hash, lifecycle, attention, + action_contract_json, reason, journal_kind, journal_json, projection_json, + evidence_refs_json, actor, drill, created_at + ) VALUES (?, ?, ?, ?, ?, 'active', 'investigate', ?, ?, 'publication', ?, ?, '["fact-a"]', 'deterministic_controller', 0, ?)`, + id, situationID, seq, seq, "sha256:material", contract, + map[bool]string{true: "first_authoritative_state", false: "attention_changed"}[seq == 1], + journal, projection, created); err != nil { + t.Fatalf("insert transition %d: %v", seq, err) + } + if _, err := st.DB().ExecContext(ctx, ` + INSERT INTO situation_transition_stream (id, transition_id, situation_id, sequence, status, created_at) + VALUES (?, ?, ?, ?, 'pending', ?)`, "stream-"+id, id, situationID, seq, created); err != nil { + t.Fatalf("insert stream row %d: %v", seq, err) + } + } + last := count + summary := fmt.Sprintf(`{"situation_id":%q,"version":%d,"source_transition_sequence":%d,"title":"Situation api",`+ + `"current_attention":"investigate","peak_attention":"investigate","investigation_work":[],`+ + `"recorded_operator_context":[],"action_contract":{"next_actor":"alertint","alertint_action":"run_acute_triage",`+ + `"alertint_status":"running","next_update_on":["triage_outcome"]},"effective_started_at":%q,`+ + `"recurrence_count":0,"updated_at":%q}`, + situationID, last, last, at.UTC().Format(time.RFC3339Nano), + at.Add(time.Duration(last)*time.Minute).UTC().Format(time.RFC3339Nano)) + if _, err := st.DB().ExecContext(ctx, ` + INSERT INTO situation_episode_summaries (situation_id, version, source_transition_sequence, summary_json, updated_at) + VALUES (?, ?, ?, ?, ?)`, situationID, last, last, summary, + at.Add(time.Duration(last)*time.Minute).UTC().Format(time.RFC3339Nano)); err != nil { + t.Fatalf("insert episode summary: %v", err) + } + return ids +} + +// seedSituationIntentForMCP writes one durable notification intent. +func seedSituationIntentForMCP(t *testing.T, st *store.Store, situationID, transitionID string, + seq int, effectClass, status string, at time.Time) { + t.Helper() + id := fmt.Sprintf("intent-%s-%s", effectClass, transitionID) + var summaryVersion any + requiresRoot := 0 + switch effectClass { + case "root_sync": + summaryVersion = seq + case "thread_append", "broadcast_handoff": + requiresRoot = 1 + } + var deliveredAt, channel, messageTS, deliveredAs any + if status == "delivered" { + deliveredAt = at.UTC().Format(time.RFC3339Nano) + channel = "C123" + messageTS = "1700000000.000100" + deliveredAs = map[bool]string{true: "root", false: "thread"}[effectClass == "root_sync"] + } + if _, err := st.DB().ExecContext(context.Background(), ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + summary_version, requires_root, main_channel_poke, client_message_id, status, + attempt_count, last_error_class, delivered_as, channel, message_ts, created_at, delivered_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + id, "idem:"+id, effectClass, situationID, transitionID, seq, summaryVersion, requiresRoot, + "client:"+id, status, 2, "timeout", deliveredAs, channel, messageTS, + at.UTC().Format(time.RFC3339Nano), deliveredAt); err != nil { + t.Fatalf("insert notification intent: %v", err) + } +} + +// TestSituationMCPHistoryToolsRegisteredAndReadOnly proves both Plan 3 read +// surfaces exist under stable names and neither declares a write. +func TestSituationMCPHistoryToolsRegisteredAndReadOnly(t *testing.T) { + st := newMCPStore(t) + s := NewServer(Config{}, st, audit.New(st.DB())) + + journalTool, journalHandler := s.toolListSituationTransitions() + if journalTool.Name != "alertint_list_situation_transitions" || journalHandler == nil { + t.Fatalf("journal tool = %q / handler nil=%v", journalTool.Name, journalHandler == nil) + } + deliveryTool, deliveryHandler := s.toolGetDeliveryState() + if deliveryTool.Name != "alertint_get_delivery_state" || deliveryHandler == nil { + t.Fatalf("delivery tool = %q / handler nil=%v", deliveryTool.Name, deliveryHandler == nil) + } + for _, tool := range []string{journalTool.Name, deliveryTool.Name} { + if strings.Contains(tool, "create") || strings.Contains(tool, "update") || strings.Contains(tool, "record") { + t.Errorf("tool %q reads as a mutation; Plan 3's MCP surface is read-only", tool) + } + } +} + +// TestSituationMCPTransitionJournalPagesByStableCursor proves the journal is +// bounded and paginated with a stable (sequence, id) cursor, oldest first, +// and that resuming from the returned cursor never repeats or skips an +// entry. +func TestSituationMCPTransitionJournalPagesByStableCursor(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-j", "service=journal", "incident_created", at) + seedSituationHistoryForMCP(t, st, situationID, at, 5) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleListSituationTransitions(context.Background(), + reqWith(map[string]any{"id": situationID, "limit": 2})) + if err != nil || res.IsError { + t.Fatalf("list transitions errored: %v %s", err, resultText(t, res)) + } + var page struct { + Transitions []struct { + ID string `json:"id"` + Sequence int `json:"sequence"` + Reason string `json:"reason"` + } `json:"transitions"` + NextCursor *struct { + Sequence int `json:"sequence"` + ID string `json:"id"` + } `json:"next_cursor"` + } + if err := json.Unmarshal([]byte(resultText(t, res)), &page); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if len(page.Transitions) != 2 || page.Transitions[0].Sequence != 1 || page.Transitions[1].Sequence != 2 { + t.Fatalf("first page = %+v, want sequences 1 and 2", page.Transitions) + } + if page.NextCursor == nil || page.NextCursor.Sequence != 2 { + t.Fatalf("next_cursor = %+v, want the last row's stable position", page.NextCursor) + } + + res2, err := s.handleListSituationTransitions(context.Background(), reqWith(map[string]any{ + "id": situationID, "limit": 10, + "cursor_sequence": page.NextCursor.Sequence, "cursor_id": page.NextCursor.ID, + })) + if err != nil || res2.IsError { + t.Fatalf("second page errored: %v %s", err, resultText(t, res2)) + } + var page2 struct { + Transitions []struct { + Sequence int `json:"sequence"` + } `json:"transitions"` + NextCursor *struct{} `json:"next_cursor"` + } + if err := json.Unmarshal([]byte(resultText(t, res2)), &page2); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if len(page2.Transitions) != 3 || page2.Transitions[0].Sequence != 3 { + t.Fatalf("second page = %+v, want sequences 3,4,5", page2.Transitions) + } + if page2.NextCursor != nil { + t.Fatalf("next_cursor = %+v on the final page, want null", page2.NextCursor) + } +} + +// TestSituationMCPGetSituationCarriesEpisodeAndDeliveryState proves the +// extended alertint_get_situation payload: the current Episode summary read +// coherently with its own source Transition, the Slack root coordinates and +// per-effect delivery status, and the withheld/superseded/delayed modes — +// with no claim owner, claim token, lease, or provider error body anywhere. +func TestSituationMCPGetSituationCarriesEpisodeAndDeliveryState(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-e", "service=episode", "incident_created", at) + ids := seedSituationHistoryForMCP(t, st, situationID, at, 2) + seedSituationIntentForMCP(t, st, situationID, ids[1], 2, "root_sync", "delivered", at) + seedSituationIntentForMCP(t, st, situationID, ids[0], 1, "thread_append", "withheld_by_operator_slack_floor", at) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleGetSituation(context.Background(), reqWith(map[string]any{"id": situationID})) + if err != nil || res.IsError { + t.Fatalf("get situation errored: %v %s", err, resultText(t, res)) + } + raw := resultText(t, res) + var payload map[string]any + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + episode, ok := payload["episode"].(map[string]any) + if !ok { + t.Fatalf("payload has no episode object: %v", payload["episode"]) + } + if episode["version"] != float64(2) { + t.Errorf("episode version = %v, want 2", episode["version"]) + } + source, ok := episode["source_transition"].(map[string]any) + if !ok { + t.Fatal("episode carries no source_transition; a summary must never be shown without the Transition it was folded from") + } + if source["sequence"] != float64(2) { + t.Errorf("source transition sequence = %v, want 2 (coherent with the summary)", source["sequence"]) + } + + delivery, ok := payload["slack_delivery"].(map[string]any) + if !ok { + t.Fatalf("payload has no slack_delivery object: %v", payload["slack_delivery"]) + } + effects, ok := delivery["effects"].([]any) + if !ok || len(effects) != 2 { + t.Fatalf("slack_delivery effects = %v, want two durable effects", delivery["effects"]) + } + first, _ := effects[0].(map[string]any) + if first["effect_class"] != "root_sync" || first["status"] != "delivered" { + t.Errorf("first effect = %v, want the delivered root projection first", first) + } + if first["channel"] != "C123" { + t.Errorf("delivered coordinates missing: %v", first) + } + second, _ := effects[1].(map[string]any) + if second["status"] != "withheld_by_operator_slack_floor" { + t.Errorf("withheld effect is not visible as a durable decision: %v", second) + } + + for _, forbidden := range []string{"claim_owner", "claim_token", "lease_expires_at", "xoxb-", "bot_token", "SELECT "} { + if strings.Contains(raw, forbidden) { + t.Errorf("get_situation payload leaks %q", forbidden) + } + } +} + +// TestSituationMCPDeliveryStateExposesBoundedOperationalFields proves the +// installation-level view: gap generation/status, replay progress, retries +// by effect class, uncertain outcomes, and the blocked backlog — bounded +// counts only, and no OTel metric instrument anywhere (R8). +func TestSituationMCPDeliveryStateExposesBoundedOperationalFields(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-d", "service=delivery", "incident_created", at) + ids := seedSituationHistoryForMCP(t, st, situationID, at, 1) + seedSituationIntentForMCP(t, st, situationID, ids[0], 1, "root_sync", "pending", at) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleGetDeliveryState(context.Background(), reqWith(map[string]any{})) + if err != nil || res.IsError { + t.Fatalf("get delivery state errored: %v %s", err, resultText(t, res)) + } + raw := resultText(t, res) + var payload struct { + Slack struct { + ConfigurationGeneration int64 `json:"configuration_generation"` + BlockedConfigurationCount int `json:"blocked_configuration_count"` + } `json:"slack"` + Delivery struct { + ByEffectClass []struct { + EffectClass string `json:"effect_class"` + Pending int `json:"pending"` + RetryAttempts int `json:"retry_attempts"` + } `json:"by_effect_class"` + UncertainOutcomes int `json:"uncertain_outcomes"` + ReplayBacklog int `json:"replay_backlog"` + OpenGapAgeSeconds *int `json:"open_gap_age_seconds"` + TransitionStreamPending int `json:"transition_stream_pending"` + } `json:"delivery"` + } + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Fatalf("invalid JSON: %v\n%s", err, raw) + } + if len(payload.Delivery.ByEffectClass) != 1 || payload.Delivery.ByEffectClass[0].EffectClass != "root_sync" { + t.Fatalf("by_effect_class = %+v, want one root_sync row", payload.Delivery.ByEffectClass) + } + if payload.Delivery.ByEffectClass[0].Pending != 1 { + t.Errorf("pending root_sync = %d, want 1", payload.Delivery.ByEffectClass[0].Pending) + } + if payload.Delivery.ByEffectClass[0].RetryAttempts != 1 { + t.Errorf("retry_attempts = %d, want 1 (attempt_count 2 minus the first attempt)", + payload.Delivery.ByEffectClass[0].RetryAttempts) + } + if payload.Delivery.UncertainOutcomes != 1 { + t.Errorf("uncertain_outcomes = %d, want 1 (the seeded timeout)", payload.Delivery.UncertainOutcomes) + } + if payload.Delivery.OpenGapAgeSeconds != nil { + t.Errorf("open_gap_age_seconds = %v with no gap open, want null", *payload.Delivery.OpenGapAgeSeconds) + } + if payload.Delivery.TransitionStreamPending != 1 { + t.Errorf("transition_stream_pending = %d, want 1", payload.Delivery.TransitionStreamPending) + } + for _, forbidden := range []string{"claim_owner", "claim_token", "lease_expires_at", "xoxb-"} { + if strings.Contains(raw, forbidden) { + t.Errorf("delivery state payload leaks %q", forbidden) + } + } +} + +// TestSituationMCPHistoryIsAbsentNotFabricated proves the honesty rule: a +// Situation with no Plan 3 history yet renders explicit nulls, never a +// reconstruction of current state as if it were history. +func TestSituationMCPHistoryIsAbsentNotFabricated(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-n", "service=none", "incident_created", at) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleGetSituation(context.Background(), reqWith(map[string]any{"id": situationID})) + if err != nil || res.IsError { + t.Fatalf("get situation errored: %v %s", err, resultText(t, res)) + } + var payload map[string]any + if err := json.Unmarshal([]byte(resultText(t, res)), &payload); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if payload["episode"] != nil { + t.Errorf("episode = %v for a Situation with no Transition, want explicit null", payload["episode"]) + } + delivery, ok := payload["slack_delivery"].(map[string]any) + if !ok { + t.Fatalf("slack_delivery = %v, want an object saying nothing is published", payload["slack_delivery"]) + } + if delivery["published"] != false { + t.Errorf("published = %v, want false", delivery["published"]) + } + + res2, err := s.handleListSituationTransitions(context.Background(), reqWith(map[string]any{"id": situationID})) + if err != nil || res2.IsError { + t.Fatalf("list transitions errored: %v %s", err, resultText(t, res2)) + } + if !strings.Contains(resultText(t, res2), `"transitions":[]`) { + t.Errorf("journal for a Situation with no history = %s, want an empty array", resultText(t, res2)) + } +} diff --git a/internal/notify/stdout/situation.go b/internal/notify/stdout/situation.go new file mode 100644 index 0000000..95da3a3 --- /dev/null +++ b/internal/notify/stdout/situation.go @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package stdout + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel/trace" + + "github.com/alertint/alertint-agent/internal/audit" + "github.com/alertint/alertint-agent/internal/situation" + "github.com/alertint/alertint-agent/internal/situation/model" + "github.com/alertint/alertint-agent/internal/store" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the stdout Transition stream. +// +// spec.md ("MCP, audit, logs, OTel, and stdout"): "Stdout emits one +// versioned machine-readable Situation Transition event after durable +// commit. It does not imply Slack delivery. Silent and withheld Situations +// still emit state, and consumers deduplicate by Transition ID." +// +// This worker is the consumer of migration 0017's +// situation_transition_stream outbox, which the same fenced controller +// commit that writes a Transition also writes one row of. It is deliberately +// the mirror image of the Situation notification worker with three +// differences, all of them consequences of stdout being a local pipe rather +// than an external provider: +// +// 1. it is never gated by the Slack Delivery gap — an unreachable Slack +// must not be able to stop the authoritative state stream; +// 2. it has no root dependency and no per-Situation head-of-queue rule, so +// one Situation's backlog never blocks another's; and +// 3. its only permanent outcome is a durable row this build cannot +// serialize at all. An unavailable writer retries, indefinitely. +// +// Delivery is at-least-once BY CONSTRUCTION: the line is written before the +// acknowledgement commits, so a crash in between replays the same +// Transition. That is the documented contract — a duplicate line is +// permitted, a lost one after a committed Transition is not — and it is why +// the line carries the Transition's immutable ID as its deduplication key. +// +// The line carries identities, closed codes, hashes, counters, and instants +// only: no journal headline or detail prose, no Assessment body, no Slack +// coordinate, and no token. A consumer that wants the prose reads it from +// MCP or Slack; stdout is the state stream. +// ---------------------------------------------------------------------- + +const ( + // TransitionStreamKind is the stable `kind` discriminator every stream + // line carries, so a consumer multiplexing this stdout with the Finding + // lines Notifier writes can tell them apart without guessing. + TransitionStreamKind = "situation.transition" + // TransitionStreamVersion is the stream envelope's own schema version. + // It changes only when a field's meaning changes, never when a purely + // additive field appears. + TransitionStreamVersion = 1 + + // auditActor identifies this worker in the hash-chained audit log. + auditActor = "situation.transition_stream" + + // streamErrorUnavailable is the bounded error class an unwritable + // stdout records; streamErrorInvalid is the one permanent class. + streamErrorUnavailable = "stdout_unavailable" + streamErrorInvalid = "invalid_transition" +) + +// AuditTransitionStreamEmitted records one durably acknowledged stdout +// line; AuditTransitionStreamFailed records the one permanent outcome this +// worker has, a durable row it cannot render. Both are aliases of +// internal/audit's own catalog, which is where the names are checked for +// completeness and non-collision. +const ( + AuditTransitionStreamEmitted = audit.KindTransitionStreamEmitted + AuditTransitionStreamFailed = audit.KindTransitionStreamFailed +) + +const ( + defaultStreamPoll = time.Second + defaultStreamLease = 120 * time.Second + defaultStreamBatch = 50 + defaultStreamRetryInitial = 2 * time.Second + defaultStreamRetryMax = 60 * time.Second +) + +// TransitionStreamStore is exactly the durable surface this worker drives. +// *store.Store implements it (asserted in the runtime wiring). +type TransitionStreamStore interface { + RecoverExpiredTransitionStreamClaims(ctx context.Context, now time.Time) (int, error) + ClaimTransitionStream(ctx context.Context, owner string, now time.Time, lease time.Duration, limit int) ([]store.TransitionStreamClaim, error) + MarkTransitionStreamDelivered(ctx context.Context, claim store.TransitionStreamClaim, now time.Time) error + RetryTransitionStreamEntry(ctx context.Context, claim store.TransitionStreamClaim, errorClass string, retryAt time.Time) error + FailTransitionStreamEntry(ctx context.Context, claim store.TransitionStreamClaim, errorClass string, now time.Time) error + ReleaseTransitionStreamClaim(ctx context.Context, claim store.TransitionStreamClaim) error +} + +// TransitionStreamAuditSink is the narrow audit-append surface this worker +// emits to. *audit.Auditor satisfies it. +type TransitionStreamAuditSink interface { + Append(ctx context.Context, actor, kind string, payload any) error +} + +// TransitionStreamConfig controls lease fencing, poll cadence, batch size, +// and the retry schedule. Like every other Plan 3 delivery ledger it has NO +// maximum-attempts field. +type TransitionStreamConfig struct { + // Owner identifies this worker to the store's lease fencing. Required. + Owner string + // Poll is how often the background loop wakes. Default 1s. + Poll time.Duration + // Lease is how long a claimed row is held. Default 120s. + Lease time.Duration + // Batch bounds one round. Default 50, clamped by the store's own page + // bound. + Batch int + // RetryInitial and RetryMax bound the exponential retry schedule. + // Defaults 2s and 60s. + RetryInitial time.Duration + RetryMax time.Duration +} + +func (c TransitionStreamConfig) withDefaults() TransitionStreamConfig { + if c.Poll <= 0 { + c.Poll = defaultStreamPoll + } + if c.Lease <= 0 { + c.Lease = defaultStreamLease + } + if c.Batch <= 0 { + c.Batch = defaultStreamBatch + } + if c.RetryInitial <= 0 { + c.RetryInitial = defaultStreamRetryInitial + } + if c.RetryMax <= 0 { + c.RetryMax = defaultStreamRetryMax + } + return c +} + +// TransitionStreamStats is the bounded counter set this worker exposes for +// logs. Plan 3 adds no OTel metric instruments (R8). +type TransitionStreamStats struct { + Claimed int64 + Emitted int64 + Retried int64 + Failed int64 + ClaimsLost int64 +} + +// transitionStreamLine is the canonical JSON object one stream row emits. +// Every field is an identity, a closed code, a hash, a counter, or an +// instant. Adding a prose field here would break the payload-absence +// contract this package's own test pins. +type transitionStreamLine struct { + Kind string `json:"kind"` + Version int `json:"version"` + EmittedAt time.Time `json:"emitted_at"` + SituationID string `json:"situation_id"` + PublicHandle string `json:"public_handle,omitempty"` + TransitionID string `json:"transition_id"` + Sequence int `json:"sequence"` + SummaryVersion int `json:"summary_version"` + InputVersion int `json:"input_version"` + MaterialFactHash string `json:"material_fact_hash"` + AssessmentID *string `json:"assessment_id,omitempty"` + Lifecycle string `json:"lifecycle"` + Attention string `json:"attention"` + NextActor string `json:"next_actor"` + NextUpdateAt *time.Time `json:"next_update_at,omitempty"` + Reason string `json:"reason"` + JournalKind string `json:"journal_kind"` + InterruptionPriority string `json:"interruption_priority,omitempty"` + Actor string `json:"actor"` + Drill bool `json:"drill"` + EvidenceRefCount int `json:"evidence_ref_count"` + RecurrenceCount int `json:"recurrence_count"` + EffectiveStartedAt time.Time `json:"effective_started_at"` + RecoveryObservedAt *time.Time `json:"recovery_observed_at,omitempty"` + TerminalAt *time.Time `json:"terminal_at,omitempty"` + TerminalReason string `json:"terminal_reason,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// TransitionStreamWorker claims durable stream rows, writes one canonical +// JSON line per Transition, and acknowledges the real outcome under the +// store's own fencing. +// +// It is safe for exactly one Start/Stop lifecycle; RunOnce may additionally +// be called directly (tests, or a one-shot drain) without ever calling +// Start. +type TransitionStreamWorker struct { + w io.Writer + store TransitionStreamStore + auditor TransitionStreamAuditSink + cfg TransitionStreamConfig + now func() time.Time + logger *slog.Logger + + wakeCh chan struct{} + stopCh chan struct{} + doneCh chan struct{} + + startOnce sync.Once + stopOnce sync.Once + started atomic.Bool + + mu sync.Mutex + // inflight holds every claim this worker currently owns, so Stop can + // release whatever the final pass was still holding (R6). + inflight map[string]store.TransitionStreamClaim + // writeMu serializes writes to w: one line must never interleave with + // another, and this writer is shared with the Finding Notifier. + writeMu sync.Mutex + + statsMu sync.Mutex + stats TransitionStreamStats +} + +// NewTransitionStreamWorker constructs the worker. w is typically os.Stdout; +// auditor may be nil; a nil clock falls back to the UTC wall clock and a nil +// logger to slog.Default. +func NewTransitionStreamWorker(w io.Writer, st TransitionStreamStore, cfg TransitionStreamConfig, + auditor TransitionStreamAuditSink, clock func() time.Time, logger *slog.Logger) *TransitionStreamWorker { + if strings.TrimSpace(cfg.Owner) == "" { + panic("notify/stdout: transition stream worker requires a non-empty owner") + } + if clock == nil { + clock = func() time.Time { return time.Now().UTC() } + } + if logger == nil { + logger = slog.Default() + } + return &TransitionStreamWorker{ + w: w, + store: st, + auditor: auditor, + cfg: cfg.withDefaults(), + now: clock, + logger: logger, + wakeCh: make(chan struct{}, 1), + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + inflight: map[string]store.TransitionStreamClaim{}, + } +} + +// Stats returns a snapshot of the worker's bounded counters. +func (w *TransitionStreamWorker) Stats() TransitionStreamStats { + w.statsMu.Lock() + defer w.statsMu.Unlock() + return w.stats +} + +func (w *TransitionStreamWorker) count(f func(*TransitionStreamStats)) { + w.statsMu.Lock() + defer w.statsMu.Unlock() + f(&w.stats) +} + +// RunOnce runs one full round: sweep abandoned leases, then claim and emit +// up to cfg.Batch rows in durable commit order. It returns how many rows it +// acknowledged an outcome for. +func (w *TransitionStreamWorker) RunOnce(ctx context.Context) (int, error) { + now := w.now().UTC() + if n, err := w.store.RecoverExpiredTransitionStreamClaims(ctx, now); err != nil { + w.logger.Error("situation: transition stream: recover expired claims failed", "err", err) + } else if n > 0 { + w.logger.Info("situation: transition stream: recovered abandoned claims", "count", n) + } + + claims, err := w.store.ClaimTransitionStream(ctx, w.cfg.Owner, now, w.cfg.Lease, w.cfg.Batch) + if err != nil { + return 0, fmt.Errorf("situation: transition stream: claim rows: %w", err) + } + handled := 0 + for _, claim := range claims { + if err := ctx.Err(); err != nil { + w.release(claim) //nolint:contextcheck // by design: release uses its own detached context; ctx is already done here + return handled, err + } + w.count(func(s *TransitionStreamStats) { s.Claimed++ }) + w.track(claim) + w.emitAndAcknowledge(ctx, claim) + w.untrack(claim) + handled++ + } + return handled, nil +} + +// emitAndAcknowledge writes one line and records its outcome under one +// span (R8: situation.transition_stream.emit, on internal/situation's tracer +// scope via situation.Tracer(), never a second scope of this package's own). +// The span starts AFTER the claim is durable and covers only the write plus +// the outcome class, so no exporter call ever happens inside a database +// transaction. +func (w *TransitionStreamWorker) emitAndAcknowledge(ctx context.Context, claim store.TransitionStreamClaim) { + startedAt := time.Now() + spanCtx, span := situation.Tracer().Start(ctx, situation.SpanTransitionStreamEmit, trace.WithAttributes( + situation.AttrSituationID.String(claim.Transition.SituationID), + situation.AttrTransitionID.String(claim.Transition.ID), + situation.AttrTransitionSequence.Int(claim.Transition.Sequence), + situation.AttrSummaryVersion.Int(claim.Transition.Sequence), + )) + defer span.End() + + emitErr := w.emit(w.w, claim) + ackErr := w.acknowledge(spanCtx, claim, emitErr) + span.SetAttributes( + situation.AttrResultClass.String(streamResultClass(emitErr, ackErr)), + situation.AttrDurationMS.Int64(time.Since(startedAt).Milliseconds()), + ) + if ackErr != nil && !errors.Is(ackErr, store.ErrTransitionStreamClaimLost) { + w.logger.Error("situation: transition stream: acknowledge failed", + append([]any{"stream_id", claim.StreamID, "transition_id", claim.Transition.ID, "err", ackErr}, + situation.SpanLogAttrs(span)...)...) + return + } + if emitErr == nil && ackErr == nil { + w.logger.Debug("situation: transition stream emitted", + append([]any{"transition_id", claim.Transition.ID, "sequence", claim.Transition.Sequence}, + situation.SpanLogAttrs(span)...)...) + } +} + +// streamResultClass maps one attempt's outcome onto the closed span result +// classes. +func streamResultClass(emitErr, ackErr error) string { + var invalid *invalidStreamPayloadError + switch { + case ackErr != nil: + return situation.StreamResultRetried + case emitErr == nil: + return situation.StreamResultEmitted + case errors.As(emitErr, &invalid): + return situation.StreamResultFailed + default: + return situation.StreamResultRetried + } +} + +// emit writes exactly one canonical JSON line for claim. A durable row this +// build cannot render at all is reported as an invalidStreamPayloadError error, +// the only permanent outcome; every other failure is an ordinary writer +// failure and retries. +func (w *TransitionStreamWorker) emit(out io.Writer, claim store.TransitionStreamClaim) error { + line, err := transitionStreamLineFrom(claim.Transition, w.now().UTC()) + if err != nil { + return err + } + encoded, err := json.Marshal(line) + if err != nil { + return &invalidStreamPayloadError{err: fmt.Errorf("notify/stdout: marshal transition stream line: %w", err)} + } + w.writeMu.Lock() + defer w.writeMu.Unlock() + if _, err := out.Write(append(encoded, '\n')); err != nil { + return fmt.Errorf("notify/stdout: write transition stream line: %w", err) + } + return nil +} + +// acknowledge records the durable outcome of one emit attempt: delivered, +// retried (an unavailable writer), or failed (an unrenderable durable row). +// It runs AFTER the line has already been written, which is what makes the +// stream at-least-once rather than at-most-once. +func (w *TransitionStreamWorker) acknowledge(ctx context.Context, claim store.TransitionStreamClaim, emitErr error) error { + now := w.now().UTC() + var invalid *invalidStreamPayloadError + switch { + case emitErr == nil: + if err := w.store.MarkTransitionStreamDelivered(ctx, claim, now); err != nil { + w.noteClaimLoss(err) + return err + } + w.count(func(s *TransitionStreamStats) { s.Emitted++ }) + w.audit(ctx, AuditTransitionStreamEmitted, claim, "") + return nil + case errors.As(emitErr, &invalid): + if err := w.store.FailTransitionStreamEntry(ctx, claim, streamErrorInvalid, now); err != nil { + w.noteClaimLoss(err) + return err + } + w.count(func(s *TransitionStreamStats) { s.Failed++ }) + w.logger.Error("situation: transition stream: durable row cannot be rendered; failed pending operator attention", + "stream_id", claim.StreamID, "transition_id", claim.Transition.ID, "err", emitErr) + w.audit(ctx, AuditTransitionStreamFailed, claim, streamErrorInvalid) + return nil + default: + delay := streamRetryDelay(claim.Transition.Sequence, w.cfg.RetryInitial, w.cfg.RetryMax) + if err := w.store.RetryTransitionStreamEntry(ctx, claim, streamErrorUnavailable, now.Add(delay)); err != nil { + w.noteClaimLoss(err) + return err + } + w.count(func(s *TransitionStreamStats) { s.Retried++ }) + w.logger.Warn("situation: transition stream: stdout write failed; retrying", + "stream_id", claim.StreamID, "transition_id", claim.Transition.ID, "err", emitErr) + return nil + } +} + +func (w *TransitionStreamWorker) noteClaimLoss(err error) { + if errors.Is(err, store.ErrTransitionStreamClaimLost) { + w.count(func(s *TransitionStreamStats) { s.ClaimsLost++ }) + } +} + +// audit appends one bounded audit row. The payload carries identities and +// closed codes only — never the Transition's journal prose. +func (w *TransitionStreamWorker) audit(ctx context.Context, kind string, claim store.TransitionStreamClaim, errorClass string) { + if w.auditor == nil { + return + } + payload := map[string]any{ + "situation_id": claim.Transition.SituationID, + "transition_id": claim.Transition.ID, + "sequence": claim.Transition.Sequence, + "stream_id": claim.StreamID, + } + if errorClass != "" { + payload["error_class"] = errorClass + } + if err := w.auditor.Append(ctx, auditActor, kind, payload); err != nil { + w.logger.Warn("situation: transition stream: audit append failed", + "transition_id", claim.Transition.ID, "err", err) + } +} + +// Start launches the background loop and returns immediately. Safe to call +// at most once; later calls are no-ops. +func (w *TransitionStreamWorker) Start(ctx context.Context) { + w.startOnce.Do(func() { + w.started.Store(true) + go w.run(ctx) + }) +} + +func (w *TransitionStreamWorker) run(ctx context.Context) { + defer close(w.doneCh) + ticker := time.NewTicker(w.cfg.Poll) + defer ticker.Stop() + for { + if _, err := w.RunOnce(ctx); err != nil && + !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + w.logger.Error("situation: transition stream: round failed", "err", err) + } + select { + case <-ctx.Done(): + return + case <-w.stopCh: + return + case <-ticker.C: + case <-w.wakeCh: + } + } +} + +// Wake nudges the background loop to run another round immediately. Never +// blocks; a coalesced Wake is harmless because due rows poll again anyway. +func (w *TransitionStreamWorker) Wake() { + select { + case w.wakeCh <- struct{}{}: + default: + } +} + +// Stop ends the background loop, then runs ONE bounded final pass under ctx +// and releases every claim still held (R6). This worker never joins Plan 2's +// shutdown drain rounds: it consumes already-committed history, and rows +// left pending are simply reclaimed at the next startup. +// +// Stopping a worker that was never started still runs the final pass and the +// release. Stop is idempotent. +func (w *TransitionStreamWorker) Stop(ctx context.Context) error { + w.stopOnce.Do(func() { close(w.stopCh) }) + + var loopErr error + if w.started.Load() { + select { + case <-w.doneCh: + case <-ctx.Done(): + loopErr = ctx.Err() + } + } + if loopErr == nil { + if _, err := w.RunOnce(ctx); err != nil && + !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + w.logger.Error("situation: transition stream: final pass failed", "err", err) + } + } + for _, claim := range w.heldClaims() { + w.release(claim) //nolint:contextcheck // by design: release uses its own detached context, so shutdown still releases when ctx is done + } + return loopErr +} + +func (w *TransitionStreamWorker) track(claim store.TransitionStreamClaim) { + w.mu.Lock() + defer w.mu.Unlock() + w.inflight[claim.StreamID] = claim +} + +func (w *TransitionStreamWorker) untrack(claim store.TransitionStreamClaim) { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.inflight, claim.StreamID) +} + +func (w *TransitionStreamWorker) heldClaims() []store.TransitionStreamClaim { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]store.TransitionStreamClaim, 0, len(w.inflight)) + for _, claim := range w.inflight { + out = append(out, claim) + } + return out +} + +// release hands one still-held claim straight back. It deliberately takes no +// caller context: it runs on the shutdown path, where that context is +// usually already done, and a canceled release would leave the row waiting +// out its whole lease. +func (w *TransitionStreamWorker) release(claim store.TransitionStreamClaim) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 5*time.Second) + defer cancel() + if err := w.store.ReleaseTransitionStreamClaim(ctx, claim); err != nil && + !errors.Is(err, store.ErrTransitionStreamClaimLost) { + w.logger.Warn("situation: transition stream: release claim failed", + "stream_id", claim.StreamID, "err", err) + } + w.untrack(claim) +} + +// invalidStreamPayloadError marks the one permanent outcome: a durable row this +// build cannot render at all, however many times it retries. +type invalidStreamPayloadError struct{ err error } + +func (e *invalidStreamPayloadError) Error() string { return e.err.Error() } +func (e *invalidStreamPayloadError) Unwrap() error { return e.err } + +// transitionStreamLineFrom projects one immutable Transition onto the +// canonical stream line. A Transition that does not satisfy its own +// Validate is a hand-corrupted durable row: it can never be rendered, so it +// is reported as permanently invalid rather than retried forever. +func transitionStreamLineFrom(t model.Transition, now time.Time) (transitionStreamLine, error) { + if err := t.Validate(); err != nil { + return transitionStreamLine{}, &invalidStreamPayloadError{err: fmt.Errorf("notify/stdout: transition stream row: %w", err)} + } + line := transitionStreamLine{ + Kind: TransitionStreamKind, + Version: TransitionStreamVersion, + EmittedAt: now, + // The Episode fold advances the summary by exactly one version per + // Transition (migration 0017's situation_episode_summaries_monotonic + // trigger) and Transition sequences are contiguous from one, so the + // Episode-summary version a Transition produced IS its sequence. + // Both are emitted because they answer different questions: the + // sequence orders history, the summary version names the projection + // a root card would render. + SituationID: t.SituationID, + TransitionID: t.ID, + Sequence: t.Sequence, + SummaryVersion: t.Sequence, + InputVersion: t.InputVersion, + MaterialFactHash: t.MaterialFactHash, + AssessmentID: t.AssessmentID, + Lifecycle: string(t.Lifecycle), + Attention: string(t.Attention), + NextActor: string(t.ActionContract.NextActor), + NextUpdateAt: t.ActionContract.NextUpdateAt, + Reason: string(t.Reason), + JournalKind: string(t.JournalKind), + Actor: string(t.Actor), + Drill: t.Drill, + EvidenceRefCount: len(t.EvidenceRefs), + RecurrenceCount: t.Journal.RecurrenceCount, + EffectiveStartedAt: t.Projection.EffectiveStartedAt, + RecoveryObservedAt: t.Projection.RecoveryObservedAt, + TerminalAt: t.Projection.TerminalAt, + CreatedAt: t.CreatedAt, + } + if t.InterruptionPriority != nil { + line.InterruptionPriority = string(*t.InterruptionPriority) + } + if t.Projection.PublicHandle != nil { + line.PublicHandle = *t.Projection.PublicHandle + } + if t.Projection.TerminalReason != nil { + line.TerminalReason = string(*t.Projection.TerminalReason) + } + return line, nil +} + +// streamRetryDelay is the bounded exponential retry schedule. It has no +// terminal value at any attempt count: an unavailable stdout is retried, not +// dead-lettered. +func streamRetryDelay(attempt int, initial, maxDelay time.Duration) time.Duration { + if attempt < 1 { + attempt = 1 + } + delay := maxDelay + if shift := attempt - 1; shift < 32 { + if scaled := initial << uint(shift); scaled > 0 && scaled < maxDelay { // #nosec G115 -- shift < 32 checked immediately above + delay = scaled + } + } + return delay +} diff --git a/internal/notify/stdout/situation_test.go b/internal/notify/stdout/situation_test.go new file mode 100644 index 0000000..f3b2f8f --- /dev/null +++ b/internal/notify/stdout/situation_test.go @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package stdout + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/alertint/alertint-agent/internal/situation" + "github.com/alertint/alertint-agent/internal/situation/model" + "github.com/alertint/alertint-agent/internal/store" +) + +// ---------------------------------------------------------------------- +// Fixtures: an in-memory transition-stream ledger with the same fencing +// contract internal/store implements, so this worker's crash/replay +// behaviour is provable without a database. +// ---------------------------------------------------------------------- + +const tsSituationID = "1f0f5a0c-0000-4000-8000-0000000000e1" + +func tsTime(t *testing.T, s string) time.Time { + t.Helper() + parsed, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatalf("parse time %q: %v", s, err) + } + return parsed.UTC() +} + +func tsTransition(t *testing.T, seq int, lifecycle model.Lifecycle, reason model.TransitionReason) model.Transition { //nolint:unparam // lifecycle is a real axis of this fixture; the terminal branch below exists for the cases that need it + t.Helper() + started := tsTime(t, "2026-09-05T10:00:00Z") + contract := model.ActionContract{NextActor: model.NextActorNone} + if !lifecycle.Terminal() { + action := model.AlertINTActionRunAcuteTriage + status := model.AlertINTStatusRunning + next := started.Add(15 * time.Minute) + contract = model.ActionContract{ + NextActor: model.NextActorAlertINT, + AlertINTAction: &action, + AlertINTStatus: &status, + NextUpdateAt: &next, + NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnTriageOutcome}, + } + } + projection := model.ProjectionFacts{ + EffectiveStartedAt: started, + EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload, + } + if lifecycle.Terminal() { + terminalAt := started.Add(time.Hour) + terminalReason := model.TerminalReasonResolutionMissing + projection.TerminalAt = &terminalAt + projection.TerminalReason = &terminalReason + } + tr := model.Transition{ + ID: "transition-" + string(rune('a'+seq)), + SituationID: tsSituationID, + Sequence: seq, + InputVersion: seq, + MaterialFactHash: "sha256:deadbeef", + Lifecycle: lifecycle, + Attention: model.AttentionInvestigate, + ActionContract: contract, + Reason: reason, + JournalKind: model.JournalPublication, + Journal: model.JournalData{ + Headline: "Checkout latency is being investigated", + Detail: "Two related alerts have been correlated into one Situation.", + OccurredAt: started, + }, + Projection: projection, + EvidenceRefs: []string{"evidence-1"}, + Actor: model.ActorDeterministicController, + CreatedAt: started.Add(time.Duration(seq) * time.Minute), + } + if err := tr.Validate(); err != nil { + t.Fatalf("fixture transition %d is invalid: %v", seq, err) + } + return tr +} + +type tsRow struct { + claim store.TransitionStreamClaim + status string + retryAt *time.Time + errClass string + delivered bool + leased bool +} + +type tsFakeStore struct { + mu sync.Mutex + rows []*tsRow + claimErr error +} + +func newTSFakeStore(transitions ...model.Transition) *tsFakeStore { + f := &tsFakeStore{} + for i, tr := range transitions { + f.rows = append(f.rows, &tsRow{ + claim: store.TransitionStreamClaim{StreamID: "stream-" + string(rune('a'+i)), Transition: tr}, + status: "pending", + }) + } + return f +} + +func (f *tsFakeStore) RecoverExpiredTransitionStreamClaims(context.Context, time.Time) (int, error) { + return 0, nil +} + +func (f *tsFakeStore) ClaimTransitionStream(_ context.Context, owner string, now time.Time, + _ time.Duration, limit int) ([]store.TransitionStreamClaim, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.claimErr != nil { + return nil, f.claimErr + } + out := []store.TransitionStreamClaim{} + for _, r := range f.rows { + if len(out) >= limit { + break + } + if r.status != "pending" || r.leased { + continue + } + if r.retryAt != nil && now.Before(*r.retryAt) { + continue + } + r.leased = true + r.claim.ClaimOwner = owner + r.claim.ClaimToken++ + out = append(out, r.claim) + } + return out, nil +} + +func (f *tsFakeStore) find(claim store.TransitionStreamClaim) (*tsRow, error) { + for _, r := range f.rows { + if r.claim.StreamID != claim.StreamID { + continue + } + if r.status != "pending" || r.claim.ClaimOwner != claim.ClaimOwner || r.claim.ClaimToken != claim.ClaimToken { + return nil, store.ErrTransitionStreamClaimLost + } + return r, nil + } + return nil, store.ErrNotFound +} + +func (f *tsFakeStore) MarkTransitionStreamDelivered(_ context.Context, claim store.TransitionStreamClaim, _ time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + r, err := f.find(claim) + if err != nil { + return err + } + r.status = "delivered" + r.delivered = true + r.leased = false + return nil +} + +func (f *tsFakeStore) RetryTransitionStreamEntry(_ context.Context, claim store.TransitionStreamClaim, + errorClass string, retryAt time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + r, err := f.find(claim) + if err != nil { + return err + } + r.errClass = errorClass + at := retryAt + r.retryAt = &at + r.leased = false + return nil +} + +func (f *tsFakeStore) FailTransitionStreamEntry(_ context.Context, claim store.TransitionStreamClaim, + errorClass string, _ time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + r, err := f.find(claim) + if err != nil { + return err + } + r.status = "failed" + r.errClass = errorClass + r.leased = false + return nil +} + +func (f *tsFakeStore) ReleaseTransitionStreamClaim(_ context.Context, claim store.TransitionStreamClaim) error { + f.mu.Lock() + defer f.mu.Unlock() + r, err := f.find(claim) + if err != nil { + return err + } + r.leased = false + return nil +} + +func (f *tsFakeStore) snapshot() []tsRow { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]tsRow, 0, len(f.rows)) + for _, r := range f.rows { + out = append(out, *r) + } + return out +} + +// failingWriter fails every write until enabled is set. +type tsFailingWriter struct { + mu sync.Mutex + fail bool + partial bool + buf bytes.Buffer +} + +func (w *tsFailingWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + switch { + case w.fail: + return 0, errors.New("stdout is unavailable") + case w.partial: + // A crash DURING the write: half the line reaches the consumer and + // the acknowledgement never happens. + n, _ := w.buf.Write(p[:len(p)/2]) + return n, errors.New("short write") + default: + return w.buf.Write(p) + } +} + +func (w *tsFailingWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} + +type tsAuditRow struct { + actor string + kind string + payload any +} + +type tsFakeAuditor struct { + mu sync.Mutex + rows []tsAuditRow +} + +func (a *tsFakeAuditor) Append(_ context.Context, actor, kind string, payload any) error { + a.mu.Lock() + defer a.mu.Unlock() + a.rows = append(a.rows, tsAuditRow{actor: actor, kind: kind, payload: payload}) + return nil +} + +func (a *tsFakeAuditor) kinds() []string { + a.mu.Lock() + defer a.mu.Unlock() + out := make([]string, 0, len(a.rows)) + for _, r := range a.rows { + out = append(out, r.kind) + } + return out +} + +func tsWorker(w *tsFailingWriter, st TransitionStreamStore, auditor TransitionStreamAuditSink) *TransitionStreamWorker { + clock := func() time.Time { return time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) } + return NewTransitionStreamWorker(w, st, TransitionStreamConfig{Owner: "test-owner"}, auditor, clock, nil) +} + +func tsLines(t *testing.T, out string) []map[string]any { + t.Helper() + lines := []map[string]any{} + for _, raw := range strings.Split(strings.TrimSpace(out), "\n") { + if strings.TrimSpace(raw) == "" { + continue + } + var line map[string]any + if err := json.Unmarshal([]byte(raw), &line); err != nil { + t.Fatalf("stdout line %q is not canonical JSON: %v", raw, err) + } + lines = append(lines, line) + } + return lines +} + +// ---------------------------------------------------------------------- +// Tests +// ---------------------------------------------------------------------- + +// TestSituationTransitionStreamEmitsOneVersionedLinePerTransition pins the +// contract: one canonical JSON object per committed Transition, carrying the +// stream envelope version, the Transition's ID and sequence, and the +// Episode-summary version that Transition produced. +func TestSituationTransitionStreamEmitsOneVersionedLinePerTransition(t *testing.T) { + st := newTSFakeStore( + tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState), + tsTransition(t, 2, model.LifecycleActive, model.ReasonAttentionChanged), + ) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + + handled, err := worker.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if handled != 2 { + t.Fatalf("handled = %d, want 2", handled) + } + + lines := tsLines(t, w.String()) + if len(lines) != 2 { + t.Fatalf("emitted %d lines, want 2: %q", len(lines), w.String()) + } + for i, line := range lines { + if line["kind"] != TransitionStreamKind { + t.Errorf("line %d kind = %v, want %q", i, line["kind"], TransitionStreamKind) + } + if line["version"] != float64(TransitionStreamVersion) { + t.Errorf("line %d version = %v, want %d", i, line["version"], TransitionStreamVersion) + } + if line["situation_id"] != tsSituationID { + t.Errorf("line %d situation_id = %v", i, line["situation_id"]) + } + wantSeq := float64(i + 1) + if line["sequence"] != wantSeq { + t.Errorf("line %d sequence = %v, want %v", i, line["sequence"], wantSeq) + } + // The Episode fold advances the summary by exactly one version per + // Transition and Transition sequences are contiguous from one, so + // the summary version a Transition produced IS its sequence. + if line["summary_version"] != wantSeq { + t.Errorf("line %d summary_version = %v, want %v", i, line["summary_version"], wantSeq) + } + if line["transition_id"] == "" || line["transition_id"] == nil { + t.Errorf("line %d carries no transition_id", i) + } + } + for _, r := range st.snapshot() { + if !r.delivered { + t.Errorf("stream row %s was never acknowledged", r.claim.StreamID) + } + } +} + +// TestSituationTransitionStreamAcknowledgesWithFencing proves the +// acknowledgement is fenced: a row whose claim moved on (a sweep reclaimed +// it mid-write) is never marked delivered by the stale holder, and the +// worker keeps going rather than failing the round. +func TestSituationTransitionStreamAcknowledgesWithFencing(t *testing.T) { + tr := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st := newTSFakeStore(tr) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + + claims, err := st.ClaimTransitionStream(context.Background(), "someone-else", time.Now().UTC(), time.Minute, 10) + if err != nil || len(claims) != 1 { + t.Fatalf("seed claim: %v, %d claims", err, len(claims)) + } + stale := claims[0] + stale.ClaimOwner = "test-owner" + stale.ClaimToken = 1 + + if err := worker.acknowledge(context.Background(), stale, nil); !errors.Is(err, store.ErrTransitionStreamClaimLost) { + t.Fatalf("acknowledge with a stale claim = %v, want ErrTransitionStreamClaimLost", err) + } + if st.snapshot()[0].delivered { + t.Fatal("a stale claim marked the row delivered") + } +} + +// TestSituationTransitionStreamCrashBeforeAcknowledgementNeverLoses proves +// the at-least-once contract from the consumer's side: the line is written +// BEFORE the acknowledgement commits, so a crash between the two replays +// the same Transition — a duplicate line, never a lost one. Consumers +// deduplicate by transition_id. +func TestSituationTransitionStreamCrashBeforeAcknowledgementNeverLoses(t *testing.T) { + tr := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st := newTSFakeStore(tr) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + + // Round one: write the line, then "crash" before acknowledging — the + // lease simply expires and the row returns to the pool. + claims, err := st.ClaimTransitionStream(context.Background(), "test-owner", time.Now().UTC(), time.Minute, 1) + if err != nil || len(claims) != 1 { + t.Fatalf("claim: %v", err) + } + if err := worker.emit(w, claims[0]); err != nil { + t.Fatalf("emit: %v", err) + } + st.rows[0].leased = false // the crashed process's lease is swept at startup + + // Round two: a fresh process replays the still-pending row. + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + lines := tsLines(t, w.String()) + if len(lines) != 2 { + t.Fatalf("emitted %d lines, want 2 (the duplicate a crash before acknowledgement permits)", len(lines)) + } + if lines[0]["transition_id"] != lines[1]["transition_id"] { + t.Fatal("the replayed line names a different transition; consumers cannot deduplicate") + } + if !st.snapshot()[0].delivered { + t.Fatal("the replayed row was never acknowledged") + } +} + +// TestSituationTransitionStreamCrashDuringWriteReplaysTheWholeLine proves a +// half-written line is never acknowledged: the row stays pending, retries, +// and the complete line is written again. +func TestSituationTransitionStreamCrashDuringWriteReplaysTheWholeLine(t *testing.T) { + tr := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st := newTSFakeStore(tr) + w := &tsFailingWriter{partial: true} + worker := tsWorker(w, st, nil) + + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + row := st.snapshot()[0] + if row.delivered { + t.Fatal("a short write was acknowledged as delivered") + } + if row.status != "pending" || row.retryAt == nil { + t.Fatalf("row after a short write = %+v, want pending with a retry time", row) + } + + w.mu.Lock() + w.partial = false + w.buf.Reset() + w.mu.Unlock() + st.rows[0].retryAt = nil + + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce (retry): %v", err) + } + if lines := tsLines(t, w.String()); len(lines) != 1 { + t.Fatalf("retry emitted %d lines, want 1 complete line", len(lines)) + } + if !st.snapshot()[0].delivered { + t.Fatal("the retried row was never acknowledged") + } +} + +// TestSituationTransitionStreamWriterFailureRetriesIndefinitely proves an +// unavailable stdout retries — never dead-letters — and touches no Slack +// intent state at all (this worker has no notification-intent surface). +func TestSituationTransitionStreamWriterFailureRetriesIndefinitely(t *testing.T) { + tr := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st := newTSFakeStore(tr) + w := &tsFailingWriter{fail: true} + worker := tsWorker(w, st, nil) + + for range 3 { + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + st.rows[0].retryAt = nil + } + row := st.snapshot()[0] + if row.status != "pending" { + t.Fatalf("status after three stdout failures = %q, want pending (retry is indefinite)", row.status) + } + if row.errClass != streamErrorUnavailable { + t.Fatalf("last_error_class = %q, want %q", row.errClass, streamErrorUnavailable) + } +} + +// TestSituationTransitionStreamInvalidPayloadFailsOnlyThatRow proves a +// durable row this build cannot serialize moves to failed — with an audit +// record — while every other row in the same batch still emits. +func TestSituationTransitionStreamInvalidPayloadFailsOnlyThatRow(t *testing.T) { + bad := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + bad.SituationID = "" // a hand-corrupted durable row + good := tsTransition(t, 2, model.LifecycleActive, model.ReasonAttentionChanged) + st := newTSFakeStore(bad, good) + w := &tsFailingWriter{} + auditor := &tsFakeAuditor{} + worker := tsWorker(w, st, auditor) + + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + rows := st.snapshot() + if rows[0].status != "failed" || rows[0].errClass != streamErrorInvalid { + t.Fatalf("invalid row = %+v, want failed/%s", rows[0], streamErrorInvalid) + } + if !rows[1].delivered { + t.Fatal("a valid row in the same batch was not emitted") + } + kinds := auditor.kinds() + found := false + for _, k := range kinds { + if k == AuditTransitionStreamFailed { + found = true + } + } + if !found { + t.Fatalf("audit kinds = %v, want one %s row", kinds, AuditTransitionStreamFailed) + } +} + +// TestSituationTransitionStreamEmitsForSilentAndWithheldSituations proves +// the stream is state, not Slack: a Situation that creates no Slack effect +// at all still emits its Transitions, and the line says nothing about Slack +// delivery. +func TestSituationTransitionStreamEmitsForSilentAndWithheldSituations(t *testing.T) { + quiet := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + quiet.JournalKind = model.JournalNone + quiet.Journal = model.JournalData{OccurredAt: quiet.CreatedAt} + st := newTSFakeStore(quiet) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + lines := tsLines(t, w.String()) + if len(lines) != 1 { + t.Fatalf("a silent Situation emitted %d lines, want 1", len(lines)) + } + for _, key := range []string{"slack_channel", "slack_message_ts", "delivered_to_slack", "channel", "message_ts"} { + if _, ok := lines[0][key]; ok { + t.Errorf("stdout line carries %q; stdout success must never imply Slack delivery", key) + } + } +} + +// TestSituationTransitionStreamLineCarriesNoProse proves the payload-absence +// contract: identities, closed codes, hashes, counters, and instants only — +// never journal headline/detail prose, an Assessment body, or a Slack +// response. +func TestSituationTransitionStreamLineCarriesNoProse(t *testing.T) { + tr := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st := newTSFakeStore(tr) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + out := w.String() + for _, prose := range []string{tr.Journal.Headline, tr.Journal.Detail} { + if strings.Contains(out, prose) { + t.Fatalf("stdout line carries prose %q", prose) + } + } +} + +// TestSituationTransitionStreamStopRunsOneFinalPassAndReleasesClaims proves +// R6 for this worker: Stop runs exactly one bounded final pass and releases +// whatever it still holds, so a shutdown leaves no committed Transition +// waiting out a lease. +func TestSituationTransitionStreamStopRunsOneFinalPassAndReleasesClaims(t *testing.T) { + tr := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st := newTSFakeStore(tr) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + + if err := worker.Stop(context.Background()); err != nil { + t.Fatalf("Stop: %v", err) + } + row := st.snapshot()[0] + if !row.delivered { + t.Fatal("Stop's final pass did not emit the pending Transition") + } + if row.leased { + t.Fatal("Stop left a claim held") + } +} + +// TestSituationTransitionStreamEmitSpanUsesTheSituationScope proves R8's +// third new span lands on internal/situation's EXISTING instrumentation +// scope (via situation.Tracer(), never a second scope of this package's +// own), carries the Transition identity attributes, and carries no prose. +func TestSituationTransitionStreamEmitSpanUsesTheSituationScope(t *testing.T) { + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sdktrace.NewSimpleSpanProcessor(exporter))) + previous := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { + otel.SetTracerProvider(previous) + _ = tp.Shutdown(context.Background()) + }) + + tr := tsTransition(t, 3, model.LifecycleActive, model.ReasonAttentionChanged) + st := newTSFakeStore(tr) + w := &tsFailingWriter{} + worker := tsWorker(w, st, nil) + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + spans := exporter.GetSpans() + if len(spans) != 1 { + t.Fatalf("recorded %d spans, want exactly 1", len(spans)) + } + span := spans[0] + if span.Name != situation.SpanTransitionStreamEmit { + t.Fatalf("span name = %q, want %q", span.Name, situation.SpanTransitionStreamEmit) + } + if got := span.InstrumentationScope.Name; got != "github.com/alertint/alertint-agent/internal/situation" { + t.Fatalf("instrumentation scope = %q, want internal/situation's existing scope", got) + } + attrs := map[string]string{} + for _, kv := range span.Attributes { + attrs[string(kv.Key)] = kv.Value.String() + } + for key, want := range map[string]string{ + string(situation.AttrSituationID): tsSituationID, + string(situation.AttrTransitionID): tr.ID, + string(situation.AttrTransitionSequence): "3", + string(situation.AttrSummaryVersion): "3", + string(situation.AttrResultClass): situation.StreamResultEmitted, + } { + if attrs[key] != want { + t.Errorf("span %s = %q, want %q", key, attrs[key], want) + } + } + for key, value := range attrs { + if !strings.HasPrefix(key, "alertint.") { + t.Errorf("span carries a non-alertint attribute %q", key) + } + if strings.Contains(value, tr.Journal.Headline) || strings.Contains(value, tr.Journal.Detail) { + t.Errorf("span attribute %q leaks journal prose", key) + } + } +} diff --git a/internal/situation/controller.go b/internal/situation/controller.go index 13147de..073a8d3 100644 --- a/internal/situation/controller.go +++ b/internal/situation/controller.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/alertint/alertint-agent/internal/llm" @@ -1084,6 +1085,7 @@ type historyBasis struct { // state warrants, so the whole reconciliation fails closed and the // Situation stays due. func (c *Controller) commit(ctx context.Context, claim Claim, basis historyBasis, commit ControllerCommit) error { + startedAt := time.Now() history, err := c.buildHistory(claim, basis, commit) if err != nil { c.logger.Error("situation: controller history derivation failed", @@ -1104,6 +1106,7 @@ func (c *Controller) commit(ctx context.Context, claim Claim, basis historyBasis return &commitFailedError{err: err} } c.auditCommitSuccess(ctx, claim, commit) + c.observeHistoryCommit(ctx, claim, commit, startedAt) // The enclosing reconcile span (Reconcile) learns what this cycle // committed: the fresh attempt's identity/derivation, if any, and the // hashes — identity and digests only. @@ -1359,6 +1362,122 @@ func (c *Controller) auditCommitSuccess(ctx context.Context, claim Claim, commit } } +// observeHistoryCommit is Plan 3 Task 9's audit + OTel surface for one +// landed commit's durable history (R8). It runs AFTER CommitController has +// returned, so no audit append or exporter call ever happens inside a +// database transaction (Global Constraint), and it records identities, +// sequences, versions, closed codes, and one duration — never journal +// prose, an Episode narrative, a proposal, or a provider body. +// +// A non-material cycle still opens the span, with result class +// "no_history": "this cycle deliberately wrote nothing" is an operational +// answer worth having, and its absence would read as a lost commit. +func (c *Controller) observeHistoryCommit(ctx context.Context, claim Claim, commit ControllerCommit, startedAt time.Time) { + _, span := tracer().Start(ctx, SpanHistoryCommit, trace.WithAttributes( + AttrSituationID.String(claim.Situation.ID), + AttrInputVersion.Int(claim.Situation.InputVersion), + )) + defer span.End() + + if commit.History == nil { + span.SetAttributes( + AttrResultClass.String(HistoryResultNoHistory), + AttrDurationMS.Int64(time.Since(startedAt).Milliseconds()), + ) + return + } + history := commit.History + attrs := []attribute.KeyValue{ + AttrResultClass.String(HistoryResultCommitted), + AttrDurationMS.Int64(time.Since(startedAt).Milliseconds()), + } + logAttrs := []any{ + "situation_id", claim.Situation.ID, + "input_version", claim.Situation.InputVersion, + "transitions", len(history.Transitions), + "intents", len(history.Intents), + } + // A commit can legitimately carry intents but NO Transition: the R4 + // deadline refresh of an already-published root is the one Slack effect + // a non-material reconciliation may create. + if len(history.Transitions) > 0 { + last := history.Transitions[len(history.Transitions)-1] + attrs = append(attrs, + AttrTransitionID.String(last.ID), + AttrTransitionSequence.Int(last.Sequence)) + logAttrs = append(logAttrs, + "latest_transition_id", last.ID, + "latest_transition_sequence", last.Sequence) + } + if history.Summary != nil { + attrs = append(attrs, AttrSummaryVersion.Int(history.Summary.Version)) + } + span.SetAttributes(attrs...) + c.logger.Info("situation: history committed", append(logAttrs, spanLogAttrs(span)...)...) + + c.auditHistoryCommit(ctx, claim, history) +} + +// auditHistoryCommit appends the bounded Plan 3 audit trail for one landed +// commit: one row per immutable Transition (with the consumed operator +// artifact's identity when the Transition is an artifact journaling, R1), +// one for the Episode-summary version folded across them, and one per +// notification intent created. Payloads carry identities and closed codes +// only — never headline/detail prose or a Slack coordinate that does not +// exist yet. +func (c *Controller) auditHistoryCommit(ctx context.Context, claim Claim, history *HistoryCommit) { + for i := range history.Transitions { + t := history.Transitions[i] + payload := map[string]any{ + "situation_id": t.SituationID, "transition_id": t.ID, "sequence": t.Sequence, + "input_version": t.InputVersion, "reason": string(t.Reason), + "journal_kind": string(t.JournalKind), "lifecycle": string(t.Lifecycle), + "attention": string(t.Attention), "actor": string(t.Actor), + "material_fact_hash": t.MaterialFactHash, "drill": t.Drill, + } + kind := auditKindTransitionCommitted + if t.OperatorArtifactInputID != nil { + kind = auditKindArtifactJournaled + payload["operator_artifact_input_id"] = *t.OperatorArtifactInputID + } + c.auditAppend(ctx, kind, payload) + } + if history.Summary != nil { + c.auditAppend(ctx, auditKindSummaryProjected, map[string]any{ + "situation_id": history.Summary.SituationID, "version": history.Summary.Version, + "source_transition_sequence": history.Summary.SourceTransitionSequence, + "current_attention": string(history.Summary.CurrentAttention), + "peak_attention": string(history.Summary.PeakAttention), + "recurrence_count": history.Summary.RecurrenceCount, + }) + } + for i := range history.Intents { + n := history.Intents[i] + payload := map[string]any{ + "situation_id": claim.Situation.ID, "intent_id": n.ID, + "effect_class": string(n.EffectClass), "status": string(n.Status), + "main_channel_poke": n.MainChannelPoke, "requires_root": n.RequiresRoot, + } + if n.InterruptionPriority != nil { + payload["interruption_priority"] = string(*n.InterruptionPriority) + } + if n.TransitionSequence != nil { + payload["transition_sequence"] = *n.TransitionSequence + } + if n.SummaryVersion != nil { + payload["summary_version"] = *n.SummaryVersion + } + kind := auditKindIntentCreated + if n.Status == model.IntentWithheld { + // A floor-withheld poke is a durable DECISION, not an absent + // row: it gets its own event so an operator can see what the + // configured floor actually suppressed. + kind = auditKindNotificationWithheld + } + c.auditAppend(ctx, kind, payload) + } +} + // Reconcile performs one claimed Situation's full reconciliation cycle: // coherent load; local fact derivation/append; Snapshot/hashes/reasons/ // floors; pure Triage decisions; deterministic/reuse check; only then diff --git a/internal/situation/notification_worker.go b/internal/situation/notification_worker.go index 3657814..7a80ad6 100644 --- a/internal/situation/notification_worker.go +++ b/internal/situation/notification_worker.go @@ -13,6 +13,8 @@ import ( "sync/atomic" "time" + "go.opentelemetry.io/otel/trace" + "github.com/alertint/alertint-agent/internal/situation/model" ) @@ -406,6 +408,59 @@ type NotificationWorker struct { stats NotificationWorkerStats statsMu sync.Mutex + + // audit is Plan 3 Task 9's durable operator trail for this ledger's + // whole lifecycle (claim, delivery, retry, configuration block, + // permanent failure, supersession, and the gap generation's own three + // events). Optional: a nil sink simply emits nothing, exactly like + // TriageWorker's own SetAuditSink seam. + audit AuditSink +} + +// SetAuditSink wires the audit trail for this worker's delivery lifecycle +// (Plan 3 Task 9). Call before Start; a nil sink leaves auditing off. +func (w *NotificationWorker) SetAuditSink(sink AuditSink) { w.audit = sink } + +// auditAppend emits one bounded audit row. Payloads carry identities, +// closed codes, and counts only — never a Slack response body, a channel +// token, a claim owner, or journal prose. +func (w *NotificationWorker) auditAppend(ctx context.Context, kind string, payload map[string]any) { + if w.audit == nil { + return + } + if err := w.audit.Append(ctx, notificationAuditActor, kind, payload); err != nil { + w.logger.Warn("situation: notification worker: audit append failed", "kind", kind, "err", err) + } +} + +// notificationAuditActor identifies this worker in the hash-chained audit +// log, alongside Plan 2's "situation.controller"/"situation.triage_worker". +const notificationAuditActor = "situation.notification_worker" + +// intentAuditPayload is the bounded identity payload every notification +// audit row starts from. +func intentAuditPayload(claim NotificationClaim) map[string]any { + payload := map[string]any{ + "intent_id": claim.Intent.ID, + "effect_class": string(claim.Intent.EffectClass), + "attempt_count": claim.Intent.AttemptCount, + } + if claim.Intent.SituationID != nil { + payload["situation_id"] = *claim.Intent.SituationID + } + if claim.Intent.TransitionID != nil { + payload["transition_id"] = *claim.Intent.TransitionID + } + if claim.Intent.TransitionSequence != nil { + payload["transition_sequence"] = *claim.Intent.TransitionSequence + } + if claim.Intent.SummaryVersion != nil { + payload["summary_version"] = *claim.Intent.SummaryVersion + } + if claim.Intent.GapGeneration != nil { + payload["gap_generation"] = *claim.Intent.GapGeneration + } + return payload } // NewNotificationWorker creates a NotificationWorker. A nil clock falls back @@ -494,6 +549,10 @@ func (w *NotificationWorker) advanceGapState(ctx context.Context, state SlackDel w.logger.Error("situation: notification worker: open delivery gap failed", "err", err) } else if opened { w.count(func(s *NotificationWorkerStats) { s.GapsOpened++ }) + w.auditAppend(ctx, auditKindGapOpened, map[string]any{ + "first_failure_at": state.FirstFailureAt.UTC().Format(time.RFC3339Nano), + "continuous_for_ms": now.Sub(*state.FirstFailureAt).Milliseconds(), + }) w.logger.Warn("situation: notification worker: slack delivery gap opened", "first_failure_at", state.FirstFailureAt.Format(time.RFC3339), "continuous_for", now.Sub(*state.FirstFailureAt).String()) @@ -505,6 +564,7 @@ func (w *NotificationWorker) advanceGapState(ctx context.Context, state SlackDel w.logger.Error("situation: notification worker: complete delivery gap failed", "err", err) } else if done { w.count(func(s *NotificationWorkerStats) { s.GapsCompleted++ }) + w.auditAppend(ctx, auditKindGapCompleted, map[string]any{"gap_generation": generation}) w.logger.Info("situation: notification worker: slack delivery gap replay complete", "gap_generation", generation) } } @@ -570,6 +630,7 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState w.logger.Error("situation: notification worker: recover delivery gap failed", "err", err) } else if recovered { w.count(func(s *NotificationWorkerStats) { s.GapsRecovered++ }) + w.auditAppend(ctx, auditKindGapRecovered, map[string]any{"gap_generation": generation}) w.logger.Warn("situation: notification worker: slack delivery recovered; replaying gap", "gap_generation", generation) } @@ -654,6 +715,35 @@ func (w *NotificationWorker) processOne(ctx context.Context, claim NotificationC w.trackClaim(claim) defer w.untrackClaim(claim) + // R8: one span per claimed intent, on Plan 2's tracer scope. It starts + // AFTER the claim is durable and wraps only the out-of-transaction Slack + // call plus the outcome class, so no exporter call ever happens inside a + // database transaction. + startedAt := time.Now() + spanCtx, span := tracer().Start(ctx, SpanNotificationDeliver, trace.WithAttributes( + AttrIntentID.String(claim.Intent.ID), + AttrIntentEffectClass.String(string(claim.Intent.EffectClass)), + AttrIntentAttempt.Int(claim.Intent.AttemptCount), + )) + defer span.End() + if claim.Intent.SituationID != nil { + span.SetAttributes(AttrSituationID.String(*claim.Intent.SituationID)) + } + if claim.Intent.TransitionSequence != nil { + span.SetAttributes(AttrTransitionSequence.Int(*claim.Intent.TransitionSequence)) + } + if claim.Intent.SummaryVersion != nil { + span.SetAttributes(AttrSummaryVersion.Int(*claim.Intent.SummaryVersion)) + } + if claim.Intent.GapGeneration != nil { + span.SetAttributes(AttrGapGeneration.String(*claim.Intent.GapGeneration)) + } + w.auditAppend(ctx, auditKindNotificationClaimed, intentAuditPayload(claim)) + defer func() { + span.SetAttributes(AttrDurationMS.Int64(time.Since(startedAt).Milliseconds())) + }() + ctx = spanCtx + deliverCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -671,6 +761,7 @@ func (w *NotificationWorker) processOne(ctx context.Context, claim NotificationC // concurrent commit superseded this root projection). Acknowledging // now would race whatever owns the row; the durable outcome is // whatever that owner writes. + span.SetAttributes(AttrResultClass.String(DeliverResultClaimLost)) w.count(func(s *NotificationWorkerStats) { s.ClaimsLost++ }) w.logger.Warn("situation: notification worker: lease lost mid-delivery; abandoning claim", "intent_id", claim.Intent.ID, "effect_class", string(claim.Intent.EffectClass)) @@ -684,14 +775,14 @@ func (w *NotificationWorker) processOne(ctx context.Context, claim NotificationC now := w.now().UTC() if deliverErr == nil { - w.acknowledgeDelivered(writeCtx, claim, delivery, now) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context + w.acknowledgeDelivered(writeCtx, claim, delivery, now, span) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context return } - w.acknowledgeFailure(writeCtx, claim, deliverErr, now) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context + w.acknowledgeFailure(writeCtx, claim, deliverErr, now, span) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context } func (w *NotificationWorker) acknowledgeDelivered(ctx context.Context, claim NotificationClaim, - delivery NotificationDelivery, now time.Time) { + delivery NotificationDelivery, now time.Time, span trace.Span) { // Slack answered, so the dependency is healthy regardless of whether // this particular intent's row was still ours to write. if err := w.store.ObserveSlackSuccess(ctx, now); err != nil { @@ -701,15 +792,29 @@ func (w *NotificationWorker) acknowledgeDelivered(ctx context.Context, claim Not switch { case err == nil: w.count(func(s *NotificationWorkerStats) { s.Delivered++ }) + span.SetAttributes(AttrResultClass.String(DeliverResultDelivered)) + payload := intentAuditPayload(claim) + // Delivered COORDINATES are durable operator history and belong in + // the trail; the Slack response body never is. + payload["delivered_as"] = delivery.DeliveredAs + payload["channel"] = delivery.Channel + payload["message_ts"] = delivery.MessageTS + w.auditAppend(ctx, auditKindNotificationDelivered, payload) + w.logger.Info("situation: notification delivered", + append([]any{"intent_id", claim.Intent.ID, "effect_class", string(claim.Intent.EffectClass), + "delivered_as", delivery.DeliveredAs}, spanLogAttrs(span)...)...) case errors.Is(err, ErrNotificationIntentSuperseded): // R4: a newer root projection replaced this one mid-flight. The // message that just went out is the older projection's; the newer // one edits the same root next round. Expected, not a failure. w.count(func(s *NotificationWorkerStats) { s.Superseded++ }) + span.SetAttributes(AttrResultClass.String(DeliverResultSuperseded)) + w.auditAppend(ctx, auditKindNotificationSuperseded, intentAuditPayload(claim)) w.logger.Info("situation: notification worker: root projection superseded mid-delivery", "intent_id", claim.Intent.ID, "situation_id", derefString(claim.Intent.SituationID)) case errors.Is(err, ErrNotificationClaimLost): w.count(func(s *NotificationWorkerStats) { s.ClaimsLost++ }) + span.SetAttributes(AttrResultClass.String(DeliverResultClaimLost)) w.logger.Warn("situation: notification worker: delivered acknowledgement lost its claim", "intent_id", claim.Intent.ID) default: @@ -718,7 +823,8 @@ func (w *NotificationWorker) acknowledgeDelivered(ctx context.Context, claim Not } } -func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim NotificationClaim, deliverErr error, now time.Time) { +func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim NotificationClaim, + deliverErr error, now time.Time, span trace.Span) { class, code, retryAfter := classifyDeliveryFailure(deliverErr) state, stateErr := w.store.GetSlackDeliveryState(ctx) @@ -731,26 +837,43 @@ func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim Notif w.observeFailure(ctx, state, code, now) } + payload := intentAuditPayload(claim) + // The bounded closed error CLASS only: never the deliverer's error text + // and never a provider response body. + payload["error_class"] = code + var ackErr error switch class { case DeliveryRetryable: ackErr = w.retryClaim(ctx, claim, code, retryAfter, now) + if ackErr == nil { + span.SetAttributes(AttrResultClass.String(DeliverResultRetried)) + w.auditAppend(ctx, auditKindNotificationRetried, payload) + } case DeliveryConfigurationBlocking: ackErr = w.store.BlockNotificationConfiguration(ctx, claim, code, now) if ackErr == nil { w.count(func(s *NotificationWorkerStats) { s.Blocked++ }) + span.SetAttributes(AttrResultClass.String(DeliverResultBlocked)) + w.auditAppend(ctx, auditKindConfigurationBlocked, payload) w.logger.Warn("situation: notification worker: slack configuration rejected the effect; blocking until corrected", - "intent_id", claim.Intent.ID, "error_class", code) + append([]any{"intent_id", claim.Intent.ID, "error_class", code}, spanLogAttrs(span)...)...) } case DeliveryInvalid: ackErr = w.store.FailNotificationIntent(ctx, claim, code, now) if ackErr == nil { w.count(func(s *NotificationWorkerStats) { s.Failed++ }) + span.SetAttributes(AttrResultClass.String(DeliverResultFailed)) + w.auditAppend(ctx, auditKindNotificationFailed, payload) w.logger.Error("situation: notification worker: invalid durable intent; failed pending operator redrive", - "intent_id", claim.Intent.ID, "error_class", code) + append([]any{"intent_id", claim.Intent.ID, "error_class", code}, spanLogAttrs(span)...)...) } default: ackErr = w.retryClaim(ctx, claim, code, retryAfter, now) + if ackErr == nil { + span.SetAttributes(AttrResultClass.String(DeliverResultRetried)) + w.auditAppend(ctx, auditKindNotificationRetried, payload) + } } switch { case ackErr == nil: diff --git a/internal/situation/notification_worker_test.go b/internal/situation/notification_worker_test.go index a5b4cb3..6020e1b 100644 --- a/internal/situation/notification_worker_test.go +++ b/internal/situation/notification_worker_test.go @@ -7,10 +7,15 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "testing" "time" + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "github.com/alertint/alertint-agent/internal/situation/model" ) @@ -821,3 +826,230 @@ func TestNotificationWorkerReactivateConfigurationIsIdempotentForStartup(t *test } }) } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the delivery span and the audit trail +// ---------------------------------------------------------------------- + +// nwSpanRecorder swaps the global TracerProvider for an in-memory recorder +// for one test. The production binary installs no provider at all; this is +// the only place one exists inside this package's own tests. +func nwSpanRecorder(t *testing.T) *tracetest.InMemoryExporter { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sdktrace.NewSimpleSpanProcessor(exporter))) + previous := otel.GetTracerProvider() + otel.SetTracerProvider(tp) + t.Cleanup(func() { + otel.SetTracerProvider(previous) + _ = tp.Shutdown(context.Background()) + }) + return exporter +} + +type nwAuditRow struct { + actor string + kind string + payload map[string]any +} + +type nwAuditSink struct { + mu sync.Mutex + rows []nwAuditRow +} + +func (a *nwAuditSink) Append(_ context.Context, actor, kind string, payload any) error { + a.mu.Lock() + defer a.mu.Unlock() + m, _ := payload.(map[string]any) + a.rows = append(a.rows, nwAuditRow{actor: actor, kind: kind, payload: m}) + return nil +} + +func (a *nwAuditSink) kinds() []string { + a.mu.Lock() + defer a.mu.Unlock() + out := make([]string, 0, len(a.rows)) + for _, r := range a.rows { + out = append(out, r.kind) + } + return out +} + +func nwHasKind(kinds []string, want string) bool { + for _, k := range kinds { + if k == want { + return true + } + } + return false +} + +// TestTelemetryNotificationDeliverSpanCarriesIntentIdentity proves R8's +// second new span: one span per claimed intent, on Plan 2's tracer scope, +// carrying the intent identity/effect class/attempt and the closed outcome +// class — and never a Slack response body, a channel token, or a claim +// owner. +func TestTelemetryNotificationDeliverSpanCarriesIntentIdentity(t *testing.T) { + exporter := nwSpanRecorder(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{batches: [][]NotificationClaim{{nwClaim("intent-span", 3)}}} + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{Channel: "C1", MessageTS: "100.1", DeliveredAs: "root"}, nil + }} + w := nwWorker(store, deliverer, now) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + var span *tracetest.SpanStub + for i, s := range exporter.GetSpans() { + if s.Name == SpanNotificationDeliver { + span = &exporter.GetSpans()[i] + } + } + if span == nil { + t.Fatalf("no %s span recorded", SpanNotificationDeliver) + } + got := map[string]string{} + for _, kv := range span.Attributes { + got[string(kv.Key)] = kv.Value.String() + if !strings.HasPrefix(string(kv.Key), "alertint.") { + t.Errorf("delivery span carries a non-alertint attribute %q", kv.Key) + } + } + for key, want := range map[string]string{ + string(AttrIntentID): "intent-span", + string(AttrIntentEffectClass): string(model.EffectRootSync), + string(AttrIntentAttempt): "3", + string(AttrResultClass): DeliverResultDelivered, + string(AttrSituationID): "sit-1", + } { + if got[key] != want { + t.Errorf("delivery span %s = %q, want %q", key, got[key], want) + } + } + for key := range got { + for _, forbidden := range []string{"claim_owner", "lease", "token", "response"} { + if strings.Contains(key, forbidden) { + t.Errorf("delivery span carries forbidden attribute %q", key) + } + } + } + // The delivered Slack coordinates are durable operator history and + // belong in the AUDIT trail, not on the span's identity attributes. + if _, ok := got["alertint.slack.channel"]; ok { + t.Error("delivery span carries a Slack channel attribute") + } +} + +// TestSituationNotificationAuditTrailCoversTheDeliveryLifecycle proves the +// worker emits the durable operator trail spec.md requires — claim, +// delivery (with its coordinates), retry, configuration block, permanent +// failure, and supersession — with bounded payloads that never carry a raw +// error, a provider body, or a claim owner. +func TestSituationNotificationAuditTrailCoversTheDeliveryLifecycle(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + + delivered := &nwAuditSink{} + store := &nwStore{batches: [][]NotificationClaim{{nwClaim("intent-ok", 1)}}} + w := nwWorker(store, &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{Channel: "C1", MessageTS: "100.1", DeliveredAs: "root"}, nil + }}, now) + w.SetAuditSink(delivered) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce (delivered): %v", err) + } + kinds := delivered.kinds() + for _, want := range []string{auditKindNotificationClaimed, auditKindNotificationDelivered} { + if !nwHasKind(kinds, want) { + t.Errorf("audit kinds = %v, want one %s row", kinds, want) + } + } + for _, row := range delivered.rows { + if row.actor != notificationAuditActor { + t.Errorf("audit actor = %q, want %q", row.actor, notificationAuditActor) + } + for key := range row.payload { + if key == "claim_owner" || key == "claim_token" || key == "error" { + t.Errorf("audit payload carries %q", key) + } + } + if row.kind == auditKindNotificationDelivered { + if row.payload["channel"] != "C1" || row.payload["delivered_as"] != "root" { + t.Errorf("delivered audit row lost its coordinates: %v", row.payload) + } + } + } + + failures := &nwAuditSink{} + store2 := &nwStore{batches: [][]NotificationClaim{ + {nwClaim("intent-retry", 1)}, + {nwClaim("intent-config", 1)}, + {nwClaim("intent-invalid", 1)}, + }} + w2 := nwWorker(store2, &nwDeliverer{deliver: func(intent model.NotificationIntent) (NotificationDelivery, error) { + switch intent.ID { + case "intent-config": + return NotificationDelivery{}, nwDeliveryError{class: DeliveryConfigurationBlocking, code: "invalid_auth"} + case "intent-invalid": + return NotificationDelivery{}, nwDeliveryError{class: DeliveryInvalid, code: "missing_text"} + default: + return NotificationDelivery{}, nwDeliveryError{class: DeliveryRetryable, code: "ratelimited"} + } + }}, now) + w2.SetAuditSink(failures) + for i := range 3 { + if _, err := w2.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce (failures) %d: %v", i, err) + } + } + kinds2 := failures.kinds() + for _, want := range []string{ + auditKindNotificationRetried, auditKindConfigurationBlocked, auditKindNotificationFailed, + } { + if !nwHasKind(kinds2, want) { + t.Errorf("failure audit kinds = %v, want one %s row", kinds2, want) + } + } + for _, row := range failures.rows { + if raw, ok := row.payload["error_class"].(string); ok && strings.ContainsAny(raw, " :") { + t.Errorf("audit error_class %q is raw error text, not a bounded class", raw) + } + } + + superseded := &nwAuditSink{} + store3 := &nwStore{ + batches: [][]NotificationClaim{{nwClaim("intent-superseded", 1)}}, + deliverAckErr: ErrNotificationIntentSuperseded, + } + w3 := nwWorker(store3, &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{Channel: "C1", MessageTS: "100.1", DeliveredAs: "root"}, nil + }}, now) + w3.SetAuditSink(superseded) + if _, err := w3.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce (superseded): %v", err) + } + if !nwHasKind(superseded.kinds(), auditKindNotificationSuperseded) { + t.Errorf("superseded audit kinds = %v, want one %s row", superseded.kinds(), auditKindNotificationSuperseded) + } +} + +// TestSituationNotificationAuditIsOptional proves a worker with no audit +// sink still delivers: auditing is an added trail, never a delivery +// precondition. +func TestSituationNotificationAuditIsOptional(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{batches: [][]NotificationClaim{{nwClaim("intent-noaudit", 1)}}} + w := nwWorker(store, &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{Channel: "C1", MessageTS: "100.1", DeliveredAs: "root"}, nil + }}, now) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + var delivered []string + store.snapshot(func(s *nwStore) { delivered = append(delivered, s.delivered...) }) + if len(delivered) != 1 { + t.Fatalf("delivered = %v, want the one intent to deliver with no audit sink wired", delivered) + } +} diff --git a/internal/situation/telemetry.go b/internal/situation/telemetry.go index 43da83d..e6aa9e5 100644 --- a/internal/situation/telemetry.go +++ b/internal/situation/telemetry.go @@ -53,6 +53,30 @@ const ( // (TriageWorker.processOne): the claim is already durable when it // starts; it measures analysis plus completion. SpanTriageAttempt = "incident.triage.attempt" + + // Plan 3 Task 9 (R8): three additional spans on this SAME scope. Plan 3 + // adds no metric instruments, no exporter, and no configuration surface + // — the operational signals spec.md lists (retries by class, open gap + // age, replay backlog, uncertain outcomes) are bounded MCP fields and + // log fields instead. See plan.md R8 for why: telemetry.otlp exports + // traces only, so a counter would be unobservable dead code and a + // metrics exporter is a config surface this plan does not own. + + // SpanHistoryCommit covers one fenced controller commit's durable + // history: the immutable Transitions, the Episode-summary version folded + // across them, the stdout stream rows, and the notification intents they + // warranted. It starts AFTER CommitController has returned, so no + // exporter call ever happens inside a database transaction. + SpanHistoryCommit = "situation.history.commit" + // SpanNotificationDeliver covers one claimed notification intent's + // single Slack call: the claim is already durable when it starts, and + // it wraps only the out-of-transaction provider I/O plus the fenced + // acknowledgement's outcome class. + SpanNotificationDeliver = "situation.notification.deliver" + // SpanTransitionStreamEmit covers one stdout Transition-stream row's + // write and acknowledgement. Emitted from internal/notify/stdout through + // Tracer(), so it lands on this same scope. + SpanTransitionStreamEmit = "situation.transition_stream.emit" ) // Attribute keys (stable). Identity, digests, counts, closed result @@ -76,8 +100,89 @@ const ( AttrEvidencePackDigest = attribute.Key("alertint.triage.evidence_pack_digest") AttrResultClass = attribute.Key("alertint.result.class") AttrDurationMS = attribute.Key("alertint.duration_ms") + + // AttrTransitionID and the six keys below are Plan 3 Task 9's (R8) + // additions: identities, sequences, versions, counts, and closed classes + // only — never journal prose, an Episode narrative, a Slack response + // body, a channel token, or a claim owner. + AttrTransitionID = attribute.Key("alertint.transition.id") + AttrTransitionSequence = attribute.Key("alertint.transition.sequence") + AttrSummaryVersion = attribute.Key("alertint.summary.version") + AttrIntentID = attribute.Key("alertint.intent.id") + AttrIntentEffectClass = attribute.Key("alertint.intent.effect_class") + AttrIntentAttempt = attribute.Key("alertint.intent.attempt") + AttrGapGeneration = attribute.Key("alertint.gap.generation") +) + +// Closed result classes for SpanHistoryCommit's AttrResultClass. +const ( + // HistoryResultCommitted means this cycle committed durable history. + HistoryResultCommitted = "committed" + // HistoryResultNoHistory means the cycle was non-material and warranted + // no Transition, Episode version, stream row, or intent at all (R4). + HistoryResultNoHistory = "no_history" +) + +// Closed result classes for SpanNotificationDeliver's AttrResultClass. +const ( + DeliverResultDelivered = "delivered" + DeliverResultRetried = "retried" + DeliverResultBlocked = "configuration_blocked" + DeliverResultFailed = "failed" + DeliverResultSuperseded = "superseded" + DeliverResultClaimLost = "claim_lost" +) + +// Closed result classes for SpanTransitionStreamEmit's AttrResultClass. +const ( + StreamResultEmitted = "emitted" + StreamResultRetried = "retried" + StreamResultFailed = "failed" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 9 audit event kinds emitted from THIS package. +// +// The catalog of record is internal/audit (audit.SituationHistoryKinds), +// where it can be checked for completeness against spec.md and for +// non-collision with Plan 2's names. This package cannot import it: the +// audit package's own in-package tests import internal/store, and +// internal/store imports this package, so internal/situation -> +// internal/audit closes an import cycle in that test binary. The values are +// therefore restated here, and telemetry_test.go's +// TestTelemetryAuditKindsMatchTheAuditCatalog — an EXTERNAL test package, +// which can import both — fails if the two ever drift apart. Emitters +// outside this package (internal/notify/stdout, cmd/alertint) use the audit +// package's constants directly. +const ( + auditKindTransitionCommitted = "situation.history.transition_committed" + auditKindSummaryProjected = "situation.history.summary_projected" + auditKindArtifactJournaled = "situation.history.artifact_journaled" + auditKindIntentCreated = "situation.notification.intent_created" + auditKindNotificationClaimed = "situation.notification.claimed" + auditKindNotificationDelivered = "situation.notification.delivered" + auditKindNotificationRetried = "situation.notification.retried" + auditKindConfigurationBlocked = "situation.notification.configuration_blocked" + auditKindNotificationFailed = "situation.notification.failed" + auditKindNotificationWithheld = "situation.notification.withheld" + auditKindNotificationSuperseded = "situation.notification.superseded" + auditKindGapOpened = "situation.notification.gap_opened" + auditKindGapRecovered = "situation.notification.gap_recovered" + auditKindGapCompleted = "situation.notification.gap_completed" ) +// AuditKindsEmittedHere lists every audit kind this package emits, for the +// external drift test described above. +func AuditKindsEmittedHere() []string { + return []string{ + auditKindTransitionCommitted, auditKindSummaryProjected, auditKindArtifactJournaled, + auditKindIntentCreated, auditKindNotificationClaimed, auditKindNotificationDelivered, + auditKindNotificationRetried, auditKindConfigurationBlocked, auditKindNotificationFailed, + auditKindNotificationWithheld, auditKindNotificationSuperseded, + auditKindGapOpened, auditKindGapRecovered, auditKindGapCompleted, + } +} + // Closed result classes for SpanControllerReconcile's AttrResultClass. // SpanAssessmentDispatch uses L2Outcome values; SpanTriageAttempt uses the // Triage completion outcomes plus clean_skip/backoff/exhausted/lease_lost @@ -92,6 +197,19 @@ func tracer() trace.Tracer { return otel.GetTracerProvider().Tracer(tracerName) } +// Tracer exposes this package's instrumentation scope so the one Plan 3 span +// site that lives outside it — the stdout Transition-stream worker in +// internal/notify/stdout, which cannot import this package's unexported +// tracer — emits on the SAME scope rather than opening a second one (R8: +// "on Plan 2's tracer scope"). It installs no provider: with nothing +// configured every span it returns is a no-op. +func Tracer() trace.Tracer { return tracer() } + +// SpanLogAttrs is spanLogAttrs, exported for the same single out-of-package +// span site, so its paired structured log line carries the identical +// trace_id/span_id pair every span site in this package writes. +func SpanLogAttrs(span trace.Span) []any { return spanLogAttrs(span) } + // spanLogAttrs returns the trace_id/span_id slog attribute pair for span, // or nil when span carries no valid span context (no provider installed), // so a log line's identity attributes stay the same whether or not export diff --git a/internal/situation/telemetry_test.go b/internal/situation/telemetry_test.go index bbbe7c9..308559e 100644 --- a/internal/situation/telemetry_test.go +++ b/internal/situation/telemetry_test.go @@ -4,6 +4,8 @@ package situation_test import ( "context" + "os" + "path/filepath" "strings" "testing" "time" @@ -13,6 +15,7 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" + "github.com/alertint/alertint-agent/internal/audit" "github.com/alertint/alertint-agent/internal/llm" "github.com/alertint/alertint-agent/internal/situation" "github.com/alertint/alertint-agent/internal/situation/model" @@ -240,3 +243,152 @@ func TestTelemetryTriageAttemptSpanCarriesAttemptIdentityAndDigests(t *testing.T } attrValue(t, a, situation.AttrDurationMS) } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9 (R8): the three additional spans, on the same scope +// ---------------------------------------------------------------------- + +// TestTelemetryHistoryCommitSpanCarriesTransitionAndSummaryIdentity drives +// one work-bearing controller cycle and proves the new +// situation.history.commit span exists on Plan 2's tracer scope, carries the +// Transition/summary identity attributes R8 names, and carries nothing that +// could be journal prose, an Episode narrative, a proposal, or SQL text. +func TestTelemetryHistoryCommitSpanCarriesTransitionAndSummaryIdentity(t *testing.T) { + exporter := installSpanRecorder(t) + + in := ctBaseSnapshotInput() + store := &fakeControllerStore{loadInput: in, beginWorkAttempt: 2, beginRetryEpoch: 1} + client := &fakeAssessmentClient{responses: []func() (llm.OneShotCompletion, error){acceptedResponse(t)}} + c := ctController(t, store, client) + if err := c.Reconcile(context.Background(), ctBaseClaim()); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + spans := spansNamed(exporter.GetSpans(), situation.SpanHistoryCommit) + if len(spans) != 1 { + t.Fatalf("history commit spans = %d, want exactly 1", len(spans)) + } + span := spans[0] + if got := attrValue(t, span, situation.AttrResultClass).AsString(); got != situation.HistoryResultCommitted { + t.Errorf("result class = %q, want %q", got, situation.HistoryResultCommitted) + } + if attrValue(t, span, situation.AttrSituationID).AsString() == "" { + t.Error("history commit span carries no situation id") + } + if attrValue(t, span, situation.AttrTransitionID).AsString() == "" { + t.Error("history commit span carries no transition id") + } + if attrValue(t, span, situation.AttrTransitionSequence).AsInt64() < 1 { + t.Error("history commit span carries no transition sequence") + } + if attrValue(t, span, situation.AttrSummaryVersion).AsInt64() < 1 { + t.Error("history commit span carries no episode summary version") + } + if attrValue(t, span, situation.AttrDurationMS).AsInt64() < 0 { + t.Error("history commit span carries a negative duration") + } + assertNoPayloadAttributes(t, span) +} + +// assertNoPayloadAttributes is the existing payload-absence pattern applied +// to a Plan 3 span: every attribute must be a bounded identity, closed code, +// digest, count, or duration — never prose, a token, a Slack response body, +// or SQL text. +func assertNoPayloadAttributes(t *testing.T, span tracetest.SpanStub) { + t.Helper() + forbiddenSubstrings := []string{"SELECT ", "INSERT ", "xoxb-", "Bearer ", "{\"blocks\"", "\"ok\":"} + for _, kv := range span.Attributes { + if !strings.HasPrefix(string(kv.Key), "alertint.") { + t.Errorf("span %q carries a non-alertint attribute %q", span.Name, kv.Key) + } + value := kv.Value.String() + if len(value) > 128 { + t.Errorf("span %q attribute %q is %d bytes; bounded identities are never that long", span.Name, kv.Key, len(value)) + } + for _, forbidden := range forbiddenSubstrings { + if strings.Contains(value, forbidden) { + t.Errorf("span %q attribute %q leaks %q", span.Name, kv.Key, forbidden) + } + } + } +} + +// TestTelemetryPlan3SpansShareTheExistingScopeAndAddNoMetrics proves R8's +// two structural rules at once: the three new span names are declared on the +// same instrumentation scope Plan 2 uses (there is exactly one tracerName in +// this package), and Plan 3 introduces no OTel metric instrument anywhere in +// the packages it touches — the operational counts live in bounded MCP +// fields and log fields instead. +func TestTelemetryPlan3SpansShareTheExistingScopeAndAddNoMetrics(t *testing.T) { + for _, name := range []string{ + situation.SpanHistoryCommit, situation.SpanNotificationDeliver, situation.SpanTransitionStreamEmit, + } { + if !strings.HasPrefix(name, "situation.") { + t.Errorf("span name %q is outside the situation.* family", name) + } + } + // The existing three must be untouched. + for name, want := range map[string]string{ + situation.SpanControllerReconcile: "situation.controller.reconcile", + situation.SpanAssessmentDispatch: "situation.assessment.dispatch", + situation.SpanTriageAttempt: "incident.triage.attempt", + } { + if name != want { + t.Errorf("existing span renamed to %q, want %q", name, want) + } + } + for _, dir := range []string{".", "../notify/stdout", "../mcp"} { + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") || strings.HasSuffix(e.Name(), "_test.go") { + continue + } + src, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatalf("read %s/%s: %v", dir, e.Name(), err) + } + for _, forbidden := range []string{"otel/metric", "GetMeterProvider", "Int64Counter", "Float64Histogram"} { + if strings.Contains(string(src), forbidden) { + t.Errorf("%s/%s references %q; R8 forbids OTel metric instruments in Plan 3", dir, e.Name(), forbidden) + } + } + } + } +} + +// TestTelemetryAuditKindsMatchTheAuditCatalog pins that the audit event +// names internal/situation emits (restated there because it cannot import +// internal/audit without closing an import cycle in that package's own test +// binary) are exactly a subset of internal/audit's catalog of record, and +// that the catalog itself covers every name any emitter uses. +func TestTelemetryAuditKindsMatchTheAuditCatalog(t *testing.T) { + catalog := map[string]bool{} + for _, kind := range audit.SituationHistoryKinds() { + catalog[kind] = true + } + for _, kind := range situation.AuditKindsEmittedHere() { + if !catalog[kind] { + t.Errorf("internal/situation emits %q, which internal/audit's catalog does not name", kind) + } + } + // The two the stdout stream worker owns complete the catalog; nothing + // else may be left unclaimed. + claimed := map[string]bool{ + audit.KindTransitionStreamEmitted: true, + audit.KindTransitionStreamFailed: true, + // R2's owner_terminal event belongs to the input-application path + // (internal/store), which records the artifact without journaling it. + audit.KindHistoryArtifactOwnerTerminal: true, + } + for _, kind := range situation.AuditKindsEmittedHere() { + claimed[kind] = true + } + for kind := range catalog { + if !claimed[kind] { + t.Errorf("catalog names %q but no emitter claims it", kind) + } + } +} diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go index 0eb6215..c7fd815 100644 --- a/internal/store/situation_history.go +++ b/internal/store/situation_history.go @@ -681,6 +681,17 @@ func scanStreamEntry(rows *sql.Rows, streamID *string) (situationmodel.Transitio return tr, nil } +// scanClaimedStreamEntry scans one +// `SELECT st.id, st.claim_token, ` row: the +// stream row's own id and current claim token, then the joined Transition. +func scanClaimedStreamEntry(rows *sql.Rows, streamID *string, claimToken *int64) (situationmodel.Transition, error) { + tr, err := scanTransition(prefixedScanner{rows: rows, prefix: []any{streamID, claimToken}}) + if err != nil { + return situationmodel.Transition{}, fmt.Errorf("store: scan claimed transition stream entry: %w", err) + } + return tr, nil +} + // prefixedScanner lets scanTransition consume a row that carries extra // leading columns (store.scanner's own shape), without duplicating its // column list. diff --git a/internal/store/situation_startup.go b/internal/store/situation_startup.go new file mode 100644 index 0000000..712287a --- /dev/null +++ b/internal/store/situation_startup.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "fmt" + "time" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the two startup-only, zero-outward-effect catch-up sweeps +// spec.md's own startup order names between Plan 1/2 reconstruction and +// Receivers starting (steps 3 and 6). +// +// Both do exactly one thing: pull a nonterminal Situation's +// next_assessment_at forward so the ordinary controller claim path picks it +// up on its next round. Neither writes history, creates a notification +// intent, or calls anything outward — startup never publishes merely +// because the binary restarted. What the controller then commits is decided +// by the same materiality and publication rules every other cycle uses, so +// a Situation whose truth has not changed produces no Transition and no +// Slack effect at all. +// ---------------------------------------------------------------------- + +// ScheduleSituationsMissingFirstTransition makes every nonterminal Situation +// that has no Transition at all due now (spec.md startup step 3: "schedules +// nonterminal Plan 2 Situations missing a first Transition"). +// +// This is the upgrade path: a Situation that Plan 2 created and reconciled +// before Plan 3's history schema existed carries authoritative current state +// with no immutable history behind it. Scheduling it lets the next ordinary +// controller cycle write its `first_authoritative_state` Transition through +// the same fenced commit every other Transition goes through. It does NOT +// fabricate history for a Situation that is already terminal — spec.md's +// "no fabricated history for pre-Plan-3 terminal state" — because a closed +// Episode is immutable and can no longer be claimed anyway. +// +// It reports how many Situations it pulled forward. +func (s *Store) ScheduleSituationsMissingFirstTransition(ctx context.Context, now time.Time) (int, error) { + nowStr := canonicalTime(now.UTC()) + res, err := s.db.ExecContext(ctx, ` + UPDATE situations + SET next_assessment_at = ? + WHERE lifecycle IN ('active','recovery_pending') + AND next_assessment_at > ? + AND NOT EXISTS (SELECT 1 FROM situation_transitions tr WHERE tr.situation_id = situations.id)`, + nowStr, nowStr) + if err != nil { + return 0, fmt.Errorf("store: schedule situations missing a first transition: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count situations scheduled for a first transition: %w", err) + } + return int(n), nil +} + +// ScheduleSituationsWithStaleRootProjection makes every nonterminal +// Situation whose one pending root projection is behind its current Episode +// summary due now (spec.md startup step 6: "performs stale-root +// supersession"). +// +// Supersession itself is never performed here, and deliberately so: +// migration 0018 requires a superseded root_sync to name the replacement +// that retired it (`CHECK ((status = 'superseded') = (replacement_intent_id +// IS NOT NULL))`), and the replacement is created only by the fenced +// controller commit that supersedes it (situation_history.go's +// insertNotificationIntentsTx). A startup sweep that flipped the status on +// its own would either violate that CHECK or invent an intent no commit +// authored. Scheduling the Situation instead makes the next ordinary commit +// author the fresh root projection AND supersede the stale one atomically, +// which is the only path that keeps "at most one pending unsuperseded +// root_sync per Situation" true. Until then the stale root simply stays +// pending: its delivery attempt fails the deliverer's own summary-version +// check retryably, which delays it and never closes the obligation. +// +// It reports how many Situations it pulled forward. +func (s *Store) ScheduleSituationsWithStaleRootProjection(ctx context.Context, now time.Time) (int, error) { + nowStr := canonicalTime(now.UTC()) + res, err := s.db.ExecContext(ctx, ` + UPDATE situations + SET next_assessment_at = ? + WHERE lifecycle IN ('active','recovery_pending') + AND next_assessment_at > ? + AND EXISTS ( + SELECT 1 + FROM notification_intents ni + JOIN situation_episode_summaries es ON es.situation_id = ni.situation_id + WHERE ni.situation_id = situations.id + AND ni.effect_class = 'root_sync' + AND ni.status = 'pending' + AND ni.summary_version < es.version)`, + nowStr, nowStr) + if err != nil { + return 0, fmt.Errorf("store: schedule situations with a stale root projection: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count situations scheduled for a stale root projection: %w", err) + } + return int(n), nil +} diff --git a/internal/store/situation_transition_stream.go b/internal/store/situation_transition_stream.go new file mode 100644 index 0000000..8a53242 --- /dev/null +++ b/internal/store/situation_transition_stream.go @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the durable stdout Transition-stream outbox's claim and +// acknowledgement lifecycle (migration 0017's situation_transition_stream). +// +// Task 4/5 already insert one row per committed Transition inside the +// fenced controller commit, and Task 5's ListPendingTransitionStream reads +// the backlog. This file adds the four writes a worker needs to consume it +// exactly like every other fenced queue in this store: lease a bounded +// batch, acknowledge the real outcome under (id, lease_owner, claim_token), +// release what a shutdown was still holding, and sweep abandoned leases at +// startup. +// +// The stream is at-least-once by construction and says so: the line is +// written to stdout BEFORE the acknowledgement commits, so a crash between +// the two replays the same Transition. Consumers deduplicate by Transition +// ID (which is immutable and unique in this table). A delivered stdout line +// says nothing about Slack — that obligation lives in notification_intents. +// ---------------------------------------------------------------------- + +// ErrTransitionStreamClaimLost means a fenced stream acknowledgement named a +// claim that is no longer the row's current one: the lease expired and was +// swept or reclaimed, or the row was released. The write changed zero rows. +var ErrTransitionStreamClaimLost = errors.New("store: transition stream claim lost") + +// TransitionStreamClaim is one leased stdout-stream row together with the +// immutable Transition it records and the fencing pair every +// acknowledgement must carry. +type TransitionStreamClaim struct { + StreamID string + Transition situationmodel.Transition + ClaimOwner string + ClaimToken int64 +} + +// ClaimTransitionStream leases up to limit due, pending stream rows in one +// immediate transaction, oldest first, and returns each one joined to its +// Transition so the worker needs no second lookup. Claiming increments +// claim_token (fencing every prior holder out) and attempt_count. It calls +// nothing outbound. +// +// Unlike notification intents, stream rows are NOT gated by the Slack +// Delivery gap and are not ordered per Situation: stdout is a local pipe +// with no root dependency and no external provider, so an outage on the +// Slack side must never stop the authoritative state stream. +func (s *Store) ClaimTransitionStream(ctx context.Context, owner string, now time.Time, + lease time.Duration, limit int) ([]TransitionStreamClaim, error) { + if strings.TrimSpace(owner) == "" || lease <= 0 || limit <= 0 { + return nil, errors.New("store: transition stream claim requires owner, positive lease, and positive limit") + } + if limit > maxSituationHistoryPage { + limit = maxSituationHistoryPage + } + now = now.UTC() + nowStr := canonicalTime(now) + leaseExpires := canonicalTime(now.Add(lease)) + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("store: begin claim transition stream: %w", err) + } + defer func() { _ = tx.Rollback() }() + + rows, err := tx.QueryContext(ctx, ` + SELECT id FROM situation_transition_stream + WHERE status = 'pending' + AND (lease_owner IS NULL OR lease_expires_at <= ?) + AND (retry_at IS NULL OR retry_at <= ?) + ORDER BY created_at ASC, situation_id ASC, sequence ASC + LIMIT ?`, nowStr, nowStr, limit) + if err != nil { + return nil, fmt.Errorf("store: read due transition stream rows: %w", err) + } + ids, err := scanStringRows(rows) + if err != nil { + return nil, fmt.Errorf("store: scan due transition stream rows: %w", err) + } + if len(ids) == 0 { + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("store: commit empty transition stream claim: %w", err) + } + return []TransitionStreamClaim{}, nil + } + + placeholders, args := inPlaceholders(ids) + updateArgs := append([]any{owner, leaseExpires}, args...) + if _, err := tx.ExecContext(ctx, ` + UPDATE situation_transition_stream + SET lease_owner = ?, lease_expires_at = ?, claim_token = claim_token + 1, attempt_count = attempt_count + 1 + WHERE id IN (`+placeholders+`)`, updateArgs...); err != nil { // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + return nil, fmt.Errorf("store: claim transition stream rows: %w", err) + } + + claims, err := loadClaimedTransitionStreamTx(ctx, tx, ids, owner) + if err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("store: commit claim transition stream: %w", err) + } + return claims, nil +} + +// loadClaimedTransitionStreamTx re-reads the just-claimed rows with their +// Transitions inside the claiming transaction, so a claim can never name a +// Transition the same read cannot see. +func loadClaimedTransitionStreamTx(ctx context.Context, tx *sql.Tx, ids []string, owner string) ([]TransitionStreamClaim, error) { + placeholders, args := inPlaceholders(ids) + rows, err := tx.QueryContext(ctx, ` + SELECT st.id, st.claim_token, `+prefixedTransitionColumns+` + FROM situation_transition_stream st + JOIN situation_transitions t ON t.id = st.transition_id + WHERE st.id IN (`+placeholders+`) + ORDER BY st.created_at ASC, st.situation_id ASC, st.sequence ASC`, args...) // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + if err != nil { + return nil, fmt.Errorf("store: read claimed transition stream rows: %w", err) + } + defer func() { _ = rows.Close() }() + + out := make([]TransitionStreamClaim, 0, len(ids)) + for rows.Next() { + claim := TransitionStreamClaim{ClaimOwner: owner} + tr, err := scanClaimedStreamEntry(rows, &claim.StreamID, &claim.ClaimToken) + if err != nil { + return nil, err + } + claim.Transition = tr + out = append(out, claim) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate claimed transition stream rows: %w", err) + } + return out, nil +} + +// MarkTransitionStreamDelivered records one fenced successful stdout write. +func (s *Store) MarkTransitionStreamDelivered(ctx context.Context, claim TransitionStreamClaim, now time.Time) error { + return s.fencedTransitionStreamAck(ctx, claim, + `status = 'delivered', delivered_at = ?, lease_owner = NULL, lease_expires_at = NULL, retry_at = NULL`, + canonicalTime(now.UTC())) +} + +// RetryTransitionStreamEntry schedules the next attempt for one row whose +// stdout write failed. Like every other Plan 3 delivery ledger it carries no +// attempt ceiling: an unavailable stdout is retried, not dead-lettered. +func (s *Store) RetryTransitionStreamEntry(ctx context.Context, claim TransitionStreamClaim, + errorClass string, retryAt time.Time) error { + if err := validateNotificationErrorClass(errorClass); err != nil { + return err + } + return s.fencedTransitionStreamAck(ctx, claim, + `last_error_class = ?, retry_at = ?, lease_owner = NULL, lease_expires_at = NULL`, + errorClass, canonicalTime(retryAt.UTC())) +} + +// FailTransitionStreamEntry moves exactly one row to failed — reserved for a +// durable payload this build cannot serialize at all, never for an +// unavailable writer. Failing one row never touches another row, another +// Situation, or any notification intent. +func (s *Store) FailTransitionStreamEntry(ctx context.Context, claim TransitionStreamClaim, + errorClass string, _ time.Time) error { + if err := validateNotificationErrorClass(errorClass); err != nil { + return err + } + return s.fencedTransitionStreamAck(ctx, claim, + `status = 'failed', last_error_class = ?, retry_at = NULL, lease_owner = NULL, lease_expires_at = NULL`, + errorClass) +} + +// ReleaseTransitionStreamClaim hands one still-held claim straight back, so a +// shutdown never leaves a committed Transition waiting out a full lease. +func (s *Store) ReleaseTransitionStreamClaim(ctx context.Context, claim TransitionStreamClaim) error { + return s.fencedTransitionStreamAck(ctx, claim, `lease_owner = NULL, lease_expires_at = NULL`) +} + +// fencedTransitionStreamAck applies one fenced lifecycle write and resolves a +// zero-row result into ErrTransitionStreamClaimLost (or ErrNotFound). +func (s *Store) fencedTransitionStreamAck(ctx context.Context, claim TransitionStreamClaim, + setClause string, args ...any) error { + if strings.TrimSpace(claim.StreamID) == "" || strings.TrimSpace(claim.ClaimOwner) == "" || claim.ClaimToken <= 0 { + return errors.New("store: transition stream acknowledgement requires a complete claim") + } + args = append(args, claim.StreamID, claim.ClaimOwner, claim.ClaimToken) + res, err := s.db.ExecContext(ctx, ` + UPDATE situation_transition_stream SET `+setClause+` + WHERE id = ? AND status = 'pending' AND lease_owner = ? AND claim_token = ?`, args...) // #nosec G202 -- setClause is a package-local constant expression; every value is bound + if err != nil { + return fmt.Errorf("store: acknowledge transition stream row: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("store: count acknowledged transition stream row: %w", err) + } + if n != 1 { + var status string + err := s.db.QueryRowContext(ctx, + `SELECT status FROM situation_transition_stream WHERE id = ?`, claim.StreamID).Scan(&status) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("store: transition stream row %s: %w", claim.StreamID, ErrNotFound) + } + if err != nil { + return fmt.Errorf("store: classify lost transition stream claim: %w", err) + } + return ErrTransitionStreamClaimLost + } + return nil +} + +// RecoverExpiredTransitionStreamClaims returns every abandoned stdout-stream +// lease to the unclaimed pool. Startup-only in production (spec.md's own +// step 2, "recovers expired notification claims"), plus the worker's own +// per-round sweep. It never changes a row's status or attempt count — only +// its lease — so a crashed process's committed Transition is simply +// claimable again. +func (s *Store) RecoverExpiredTransitionStreamClaims(ctx context.Context, now time.Time) (int, error) { + res, err := s.db.ExecContext(ctx, ` + UPDATE situation_transition_stream + SET lease_owner = NULL, lease_expires_at = NULL + WHERE status = 'pending' AND lease_owner IS NOT NULL AND lease_expires_at <= ?`, + canonicalTime(now.UTC())) + if err != nil { + return 0, fmt.Errorf("store: recover expired transition stream claims: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: count recovered transition stream claims: %w", err) + } + return int(n), nil +} diff --git a/internal/store/situation_transition_stream_test.go b/internal/store/situation_transition_stream_test.go new file mode 100644 index 0000000..0d9cf8c --- /dev/null +++ b/internal/store/situation_transition_stream_test.go @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "errors" + "testing" + "time" +) + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: the stdout Transition-stream claim/acknowledgement +// lifecycle and the two startup catch-up sweeps, against a real database. +// ---------------------------------------------------------------------- + +// stsPendingStreamRows reads the pending stream rows for one Situation, in +// sequence order. +func stsPendingStreamRows(t *testing.T, st *Store, situationID string) []struct { + ID string + Status string +} { + t.Helper() + rows, err := st.db.QueryContext(context.Background(), + `SELECT id, status FROM situation_transition_stream WHERE situation_id = ? ORDER BY sequence ASC`, situationID) + if err != nil { + t.Fatalf("read stream rows: %v", err) + } + defer func() { _ = rows.Close() }() + var out []struct { + ID string + Status string + } + for rows.Next() { + var r struct { + ID string + Status string + } + if err := rows.Scan(&r.ID, &r.Status); err != nil { + t.Fatalf("scan stream row: %v", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate stream rows: %v", err) + } + return out +} + +// TestTransitionStreamClaimJoinsItsTransitionAndFencesAcknowledgement proves +// the whole fenced lifecycle: a claim carries the immutable Transition it +// records, a second claimant fences the first out, and a stale holder's +// acknowledgement changes nothing. +func TestTransitionStreamClaimJoinsItsTransitionAndFencesAcknowledgement(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := shSeedTwoCommitHistory(t, st, "service=stream", now) + + claims, err := st.ClaimTransitionStream(ctx, "stdout-a", now, time.Minute, 10) + if err != nil { + t.Fatalf("ClaimTransitionStream: %v", err) + } + if len(claims) != 2 { + t.Fatalf("claimed %d stream rows, want 2 (one per committed Transition)", len(claims)) + } + for i, c := range claims { + if c.Transition.SituationID != sitID { + t.Errorf("claim %d names situation %q, want %q", i, c.Transition.SituationID, sitID) + } + if c.Transition.Sequence != i+1 { + t.Errorf("claim %d transition sequence = %d, want %d (durable commit order)", i, c.Transition.Sequence, i+1) + } + if c.ClaimOwner != "stdout-a" || c.ClaimToken < 1 { + t.Errorf("claim %d fencing pair = (%q, %d)", i, c.ClaimOwner, c.ClaimToken) + } + } + + // A held lease is not re-claimable until it expires. + again, err := st.ClaimTransitionStream(ctx, "stdout-b", now, time.Minute, 10) + if err != nil { + t.Fatalf("second ClaimTransitionStream: %v", err) + } + if len(again) != 0 { + t.Fatalf("a second worker claimed %d already-leased rows, want 0", len(again)) + } + + // The lease expires and a second worker takes it; the first worker's + // acknowledgement must now change nothing at all. + later := now.Add(2 * time.Minute) + reclaimed, err := st.ClaimTransitionStream(ctx, "stdout-b", later, time.Minute, 1) + if err != nil || len(reclaimed) != 1 { + t.Fatalf("reclaim after lease expiry = %d rows, %v", len(reclaimed), err) + } + stale := claims[0] + if err := st.MarkTransitionStreamDelivered(ctx, stale, later); !errors.Is(err, ErrTransitionStreamClaimLost) { + t.Fatalf("stale acknowledgement = %v, want ErrTransitionStreamClaimLost", err) + } + if rows := stsPendingStreamRows(t, st, sitID); rows[0].Status != "pending" { + t.Fatalf("stale acknowledgement moved the row to %q", rows[0].Status) + } + + // The current holder's acknowledgement lands. + if err := st.MarkTransitionStreamDelivered(ctx, reclaimed[0], later); err != nil { + t.Fatalf("MarkTransitionStreamDelivered: %v", err) + } + if rows := stsPendingStreamRows(t, st, sitID); rows[0].Status != "delivered" { + t.Fatalf("row status = %q, want delivered", rows[0].Status) + } +} + +// TestTransitionStreamRetryFailAndRecoverExpiredClaims proves the three +// remaining acknowledgements: a retry returns the row to the pool with a +// bounded error class and a future retry time (never a terminal status), a +// failure moves ONLY that row, and the startup sweep returns an abandoned +// lease without touching status or attempts. +func TestTransitionStreamRetryFailAndRecoverExpiredClaims(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := shSeedTwoCommitHistory(t, st, "service=stream-retry", now) + + claims, err := st.ClaimTransitionStream(ctx, "stdout-a", now, time.Minute, 10) + if err != nil || len(claims) != 2 { + t.Fatalf("claim: %d rows, %v", len(claims), err) + } + if err := st.RetryTransitionStreamEntry(ctx, claims[0], "stdout_unavailable", now.Add(30*time.Second)); err != nil { + t.Fatalf("RetryTransitionStreamEntry: %v", err) + } + if err := st.FailTransitionStreamEntry(ctx, claims[1], "invalid_transition", now); err != nil { + t.Fatalf("FailTransitionStreamEntry: %v", err) + } + rows := stsPendingStreamRows(t, st, sitID) + if rows[0].Status != "pending" { + t.Errorf("retried row status = %q, want pending (retry is indefinite)", rows[0].Status) + } + if rows[1].Status != "failed" { + t.Errorf("failed row status = %q, want failed", rows[1].Status) + } + + // Not due yet, then due. + if due, err := st.ClaimTransitionStream(ctx, "stdout-a", now.Add(10*time.Second), time.Minute, 10); err != nil || len(due) != 0 { + t.Fatalf("claim before retry_at = %d rows, %v; want none", len(due), err) + } + due, err := st.ClaimTransitionStream(ctx, "stdout-a", now.Add(time.Minute), time.Minute, 10) + if err != nil || len(due) != 1 { + t.Fatalf("claim after retry_at = %d rows, %v; want 1", len(due), err) + } + + // An abandoned lease is swept without changing status or attempt count. + var attemptsBefore int + if err := st.db.QueryRowContext(ctx, + `SELECT attempt_count FROM situation_transition_stream WHERE id = ?`, due[0].StreamID).Scan(&attemptsBefore); err != nil { + t.Fatal(err) + } + recovered, err := st.RecoverExpiredTransitionStreamClaims(ctx, now.Add(10*time.Minute)) + if err != nil { + t.Fatalf("RecoverExpiredTransitionStreamClaims: %v", err) + } + if recovered != 1 { + t.Fatalf("recovered = %d, want 1", recovered) + } + var status string + var attemptsAfter int + var owner *string + if err := st.db.QueryRowContext(ctx, + `SELECT status, attempt_count, lease_owner FROM situation_transition_stream WHERE id = ?`, + due[0].StreamID).Scan(&status, &attemptsAfter, &owner); err != nil { + t.Fatal(err) + } + if status != "pending" || attemptsAfter != attemptsBefore || owner != nil { + t.Fatalf("after sweep: status=%q attempts=%d owner=%v; want pending/%d/nil", status, attemptsAfter, owner, attemptsBefore) + } +} + +// TestTransitionStreamReleaseHandsTheClaimBack proves shutdown's release +// path leaves the row immediately claimable rather than waiting out a lease. +func TestTransitionStreamReleaseHandsTheClaimBack(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + shSeedTwoCommitHistory(t, st, "service=stream-release", now) + + claims, err := st.ClaimTransitionStream(ctx, "stdout-a", now, time.Hour, 1) + if err != nil || len(claims) != 1 { + t.Fatalf("claim: %d rows, %v", len(claims), err) + } + if err := st.ReleaseTransitionStreamClaim(ctx, claims[0]); err != nil { + t.Fatalf("ReleaseTransitionStreamClaim: %v", err) + } + again, err := st.ClaimTransitionStream(ctx, "stdout-b", now, time.Hour, 1) + if err != nil || len(again) != 1 { + t.Fatalf("re-claim after release = %d rows, %v; want 1 immediately", len(again), err) + } +} + +// TestScheduleSituationsMissingFirstTransitionPullsOnlyNonterminalOnes is +// spec.md's startup step 3: a nonterminal Situation with no Transition at +// all becomes due now, one that already has history is left alone, and no +// history is ever fabricated. +func TestScheduleSituationsMissingFirstTransitionPullsOnlyNonterminalOnes(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + + bare := newSituationForGroup(t, st, "service=bare", now) + if _, err := st.db.ExecContext(ctx, + `UPDATE situations SET next_assessment_at = ? WHERE id = ?`, + canonicalTime(now.Add(time.Hour)), bare); err != nil { + t.Fatal(err) + } + withHistory, _ := shSeedTwoCommitHistory(t, st, "service=has-history", now) + if _, err := st.db.ExecContext(ctx, + `UPDATE situations SET next_assessment_at = ? WHERE id = ?`, + canonicalTime(now.Add(time.Hour)), withHistory); err != nil { + t.Fatal(err) + } + + scheduled, err := st.ScheduleSituationsMissingFirstTransition(ctx, now) + if err != nil { + t.Fatalf("ScheduleSituationsMissingFirstTransition: %v", err) + } + if scheduled != 1 { + t.Fatalf("scheduled = %d, want exactly the one Situation with no Transition", scheduled) + } + for id, wantDue := range map[string]bool{bare: true, withHistory: false} { + var next string + if err := st.db.QueryRowContext(ctx, `SELECT next_assessment_at FROM situations WHERE id = ?`, id).Scan(&next); err != nil { + t.Fatal(err) + } + if (next == canonicalTime(now)) != wantDue { + t.Errorf("situation %s next_assessment_at = %q, wantDue=%v", id, next, wantDue) + } + } + // It publishes nothing: no Transition, summary, stream row, or intent + // appeared for the bare Situation. + for _, table := range []string{"situation_transitions", "situation_episode_summaries", "situation_transition_stream", "notification_intents"} { + var n int + if err := st.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM `+table+` WHERE situation_id = ?`, bare).Scan(&n); err != nil { // #nosec G202 -- table is a package-local constant list + t.Fatal(err) + } + if n != 0 { + t.Errorf("scheduling created %d %s rows; a startup sweep must publish nothing", n, table) + } + } + // Idempotent: a second pass finds nothing left to pull forward. + if again, err := st.ScheduleSituationsMissingFirstTransition(ctx, now); err != nil || again != 0 { + t.Fatalf("second pass scheduled %d, %v; want 0", again, err) + } +} + +// TestScheduleSituationsWithStaleRootProjectionMakesTheOwnerDue is spec.md's +// startup step 6: a pending root projection behind its Situation's current +// Episode summary makes that Situation due, so the next ordinary commit +// authors the fresh root AND supersedes the stale one atomically. The sweep +// never flips a status itself — migration 0018 requires a superseded root to +// name the replacement that retired it, and only a fenced commit can author +// one. +func TestScheduleSituationsWithStaleRootProjectionMakesTheOwnerDue(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := shSeedTwoCommitHistory(t, st, "service=stale-root", now) + // A second Situation whose pending root projection is CURRENT: the sweep + // must leave it alone, which is what makes the count below meaningful. + currentID, _ := shSeedTwoCommitHistory(t, st, "service=current-root", now) + if _, err := st.db.ExecContext(ctx, + `UPDATE situations SET next_assessment_at = ? WHERE id = ?`, + canonicalTime(now.Add(time.Hour)), currentID); err != nil { + t.Fatal(err) + } + + var intentID string + var intentVersion int + if err := st.db.QueryRowContext(ctx, + `SELECT id, summary_version FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, + sitID).Scan(&intentID, &intentVersion); err != nil { + t.Fatalf("the seeded commits left no pending root projection to age: %v", err) + } + + // Advance the Episode summary past that projection by appending a third + // Transition and folding it — the only move migration 0017's monotonic + // trigger allows (version + 1, strictly later source sequence). + created := canonicalTime(now.Add(2 * time.Minute)) + if _, err := st.db.ExecContext(ctx, ` + INSERT INTO situation_transitions ( + id, situation_id, sequence, input_version, material_fact_hash, lifecycle, attention, + action_contract_json, reason, journal_kind, journal_json, projection_json, + evidence_refs_json, actor, drill, created_at + ) SELECT 'tr-stale-3', situation_id, 3, input_version, material_fact_hash, lifecycle, attention, + action_contract_json, reason, journal_kind, journal_json, projection_json, + evidence_refs_json, actor, drill, ? + FROM situation_transitions WHERE situation_id = ? AND sequence = 2`, created, sitID); err != nil { + t.Fatalf("append the third transition: %v", err) + } + if _, err := st.db.ExecContext(ctx, ` + UPDATE situation_episode_summaries + SET version = version + 1, source_transition_sequence = 3, updated_at = ? + WHERE situation_id = ?`, created, sitID); err != nil { + t.Fatalf("advance the episode summary: %v", err) + } + if _, err := st.db.ExecContext(ctx, + `UPDATE situations SET next_assessment_at = ? WHERE id = ?`, + canonicalTime(now.Add(time.Hour)), sitID); err != nil { + t.Fatal(err) + } + + n, err := st.ScheduleSituationsWithStaleRootProjection(ctx, now) + if err != nil { + t.Fatalf("ScheduleSituationsWithStaleRootProjection: %v", err) + } + if n != 1 { + t.Fatalf("scheduled = %d, want 1 (the Situation whose pending root projection is behind its summary)", n) + } + var next, status string + if err := st.db.QueryRowContext(ctx, `SELECT next_assessment_at FROM situations WHERE id = ?`, sitID).Scan(&next); err != nil { + t.Fatal(err) + } + if next != canonicalTime(now) { + t.Errorf("next_assessment_at = %q, want it pulled forward to now", next) + } + if err := st.db.QueryRowContext(ctx, `SELECT status FROM notification_intents WHERE id = ?`, intentID).Scan(&status); err != nil { + t.Fatal(err) + } + if status != "pending" { + t.Fatalf("the sweep changed the intent's status to %q; only a fenced commit may supersede a root projection", status) + } + _ = intentVersion + + // The Situation whose projection is current was never pulled forward. + var untouched string + if err := st.db.QueryRowContext(ctx, `SELECT next_assessment_at FROM situations WHERE id = ?`, currentID).Scan(&untouched); err != nil { + t.Fatal(err) + } + if untouched == canonicalTime(now) { + t.Error("the sweep pulled a Situation whose root projection is current forward") + } + // Idempotent: the stale Situation is already due, so a second pass finds + // nothing left to schedule. + if again, err := st.ScheduleSituationsWithStaleRootProjection(ctx, now); err != nil || again != 0 { + t.Fatalf("second pass scheduled %d, %v; want 0", again, err) + } +} + +// TestNotificationDeliveryStatsAreBoundedCounts proves the installation-level +// delivery snapshot MCP reads is counts only, and that its subsumed-blocked +// field lets the blocked backlog an operator can act on reach zero. +func TestNotificationDeliveryStatsAreBoundedCounts(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := shSeedTwoCommitHistory(t, st, "service=stats", now) + + stats, err := st.GetNotificationDeliveryStats(ctx, now) + if err != nil { + t.Fatalf("GetNotificationDeliveryStats: %v", err) + } + if stats.OpenGapAgeSeconds != nil { + t.Errorf("open_gap_age_seconds = %v with no gap, want nil", *stats.OpenGapAgeSeconds) + } + if stats.ReplayBacklog != 0 || stats.UncertainOutcomes != 0 { + t.Errorf("stats = %+v, want zero backlog and zero uncertain outcomes", stats) + } + if stats.TransitionStreamPending != 2 { + t.Errorf("transition_stream_pending = %d, want 2 (one row per committed Transition)", stats.TransitionStreamPending) + } + + intents, err := st.ListSituationNotificationIntents(ctx, sitID, 0) + if err != nil { + t.Fatalf("ListSituationNotificationIntents: %v", err) + } + if len(intents) == 0 { + t.Fatal("the seeded commits warranted no notification intent at all") + } + if intents[0].EffectClass != "root_sync" { + t.Errorf("first intent = %q, want the root projection first", intents[0].EffectClass) + } +} diff --git a/internal/store/situation_views.go b/internal/store/situation_views.go index b403045..145bfed 100644 --- a/internal/store/situation_views.go +++ b/internal/store/situation_views.go @@ -535,3 +535,241 @@ func (s *Store) ListPendingTransitionStream(ctx context.Context, limit int) ([]P } return out, nil } + +// ---------------------------------------------------------------------- +// Plan 3 Task 9: bounded delivery read views. MCP's read-only history and +// delivery surfaces read ONLY these — never the ledger tables directly, and +// never a claim owner, claim token, lease, Slack token, or provider error +// body. Every count below is derived from durable columns; Plan 3 adds no +// OTel metric instruments (R8), so these bounded fields plus the worker's +// log lines are the whole operational signal. +// ---------------------------------------------------------------------- + +// ListSituationNotificationIntents reads one Situation's durable delivery +// obligations in the order spec.md's own ordering rules read them: root +// projection first, then Transition sequence, then id. limit is clamped to +// maxSituationHistoryPage. +func (s *Store) ListSituationNotificationIntents(ctx context.Context, situationID string, limit int) ([]situationmodel.NotificationIntent, error) { + if strings.TrimSpace(situationID) == "" { + return nil, errors.New("store: situation notification intents read requires a situation id") + } + if limit <= 0 || limit > maxSituationHistoryPage { + limit = maxSituationHistoryPage + } + rows, err := s.db.QueryContext(ctx, ` + SELECT `+notificationIntentColumns+` + FROM notification_intents + WHERE situation_id = ? + ORDER BY (effect_class = 'root_sync') DESC, transition_sequence ASC, id ASC + LIMIT ?`, situationID, limit) + if err != nil { + return nil, fmt.Errorf("store: list situation notification intents: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []situationmodel.NotificationIntent{} + for rows.Next() { + intent, err := scanNotificationIntent(rows) + if err != nil { + return nil, fmt.Errorf("store: scan situation notification intent: %w", err) + } + out = append(out, intent) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate situation notification intents: %w", err) + } + return out, nil +} + +// EffectClassDeliveryStats is one effect class's bounded delivery counters. +type EffectClassDeliveryStats struct { + EffectClass string `json:"effect_class"` + Pending int `json:"pending"` + Retrying int `json:"retrying"` + Delivered int `json:"delivered"` + Blocked int `json:"blocked_configuration"` + Failed int `json:"failed"` + Withheld int `json:"withheld_by_operator_slack_floor"` + Superseded int `json:"superseded"` + RetryAttempts int `json:"retry_attempts"` +} + +// NotificationDeliveryStats is the installation-level bounded delivery +// snapshot MCP exposes: retries by effect class, open gap age, replay +// backlog, and uncertain-outcome counts (spec.md "MCP, audit, logs, OTel, +// and stdout"; R8 keeps them fields, never metric instruments). +type NotificationDeliveryStats struct { + ByEffectClass []EffectClassDeliveryStats `json:"by_effect_class"` + // OpenGapAgeSeconds is how long the currently open-or-replaying gap + // generation has existed, nil when there is none. + OpenGapAgeSeconds *int64 `json:"open_gap_age_seconds"` + // ReplayBacklog is how many Situation-scoped intents a currently + // replaying generation still has to deliver — the same "existed when + // this generation recovered" definition the claim gate uses. Zero when + // no generation is replaying. + ReplayBacklog int `json:"replay_backlog"` + // UncertainOutcomes counts intents whose last recorded failure class is + // one where Slack's answer did not prove either success or failure + // (transport error, timeout, undecodable response). It is the durable + // trace of ADR-0049's accepted rare external-duplicate risk. + UncertainOutcomes int `json:"uncertain_outcomes"` + // BlockedConfigurationCount counts intents held in + // blocked_configuration. BlockedConfigurationSubsumed is how many of + // those are stale root projections a newer pending root projection for + // the same Situation already replaces — they can never be reactivated + // into a delivery, so an operator reading the first number needs the + // second to know how much of it is actionable. + BlockedConfigurationCount int `json:"blocked_configuration_count"` + BlockedConfigurationSubsumed int `json:"blocked_configuration_subsumed"` + // TransitionStreamPending/Failed are the stdout Transition stream's own + // backlog. Stdout delivery is independent of Slack: a pending stream row + // says nothing about Slack, and a delivered one implies no Slack effect. + TransitionStreamPending int `json:"transition_stream_pending"` + TransitionStreamFailed int `json:"transition_stream_failed"` +} + +// uncertainDeliveryErrorClasses are the bounded last_error_class values that +// mean "Slack's answer did not prove either outcome" — the exact codes +// internal/notify/slack's client records for a transport failure, a timeout, +// and a 2xx body it could not decode. +var uncertainDeliveryErrorClasses = []string{"transport_error", "timeout", "undecodable_response"} + +// GetNotificationDeliveryStats reads the bounded installation-level delivery +// snapshot in one snapshot transaction, so its counters can never disagree +// with each other. It returns counts only — never an intent body, a claim +// owner, a Slack coordinate, or a provider error body. +func (s *Store) GetNotificationDeliveryStats(ctx context.Context, now time.Time) (NotificationDeliveryStats, error) { + nowStr := canonicalTime(now.UTC()) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return NotificationDeliveryStats{}, fmt.Errorf("store: begin notification delivery stats: %w", err) + } + defer func() { _ = tx.Rollback() }() + + stats := NotificationDeliveryStats{ByEffectClass: []EffectClassDeliveryStats{}} + if err := readEffectClassStatsTx(ctx, tx, nowStr, &stats); err != nil { + return NotificationDeliveryStats{}, err + } + if err := readGapStatsTx(ctx, tx, now, &stats); err != nil { + return NotificationDeliveryStats{}, err + } + if err := readUncertainAndBlockedStatsTx(ctx, tx, &stats); err != nil { + return NotificationDeliveryStats{}, err + } + if err := tx.QueryRowContext(ctx, ` + SELECT COALESCE(SUM(status = 'pending'), 0), COALESCE(SUM(status = 'failed'), 0) + FROM situation_transition_stream`). + Scan(&stats.TransitionStreamPending, &stats.TransitionStreamFailed); err != nil { + return NotificationDeliveryStats{}, fmt.Errorf("store: count transition stream backlog: %w", err) + } + if err := tx.Commit(); err != nil { + return NotificationDeliveryStats{}, fmt.Errorf("store: commit notification delivery stats: %w", err) + } + return stats, nil +} + +func readEffectClassStatsTx(ctx context.Context, tx *sql.Tx, nowStr string, stats *NotificationDeliveryStats) error { + rows, err := tx.QueryContext(ctx, ` + SELECT effect_class, + COALESCE(SUM(status = 'pending'), 0), + COALESCE(SUM(status = 'pending' AND retry_at IS NOT NULL AND retry_at > ?), 0), + COALESCE(SUM(status = 'delivered'), 0), + COALESCE(SUM(status = 'blocked_configuration'), 0), + COALESCE(SUM(status = 'failed'), 0), + COALESCE(SUM(status = 'withheld_by_operator_slack_floor'), 0), + COALESCE(SUM(status = 'superseded'), 0), + COALESCE(SUM(MAX(attempt_count - 1, 0)), 0) + FROM notification_intents + GROUP BY effect_class + ORDER BY effect_class ASC`, nowStr) + if err != nil { + return fmt.Errorf("store: count notification intents by effect class: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var e EffectClassDeliveryStats + if err := rows.Scan(&e.EffectClass, &e.Pending, &e.Retrying, &e.Delivered, &e.Blocked, + &e.Failed, &e.Withheld, &e.Superseded, &e.RetryAttempts); err != nil { + return fmt.Errorf("store: scan notification intent counts: %w", err) + } + stats.ByEffectClass = append(stats.ByEffectClass, e) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("store: iterate notification intent counts: %w", err) + } + return nil +} + +func readGapStatsTx(ctx context.Context, tx *sql.Tx, now time.Time, stats *NotificationDeliveryStats) error { + var openedAt, recoveredAt sql.NullString + err := tx.QueryRowContext(ctx, ` + SELECT g.opened_at, g.recovered_at + FROM slack_delivery_gaps g + JOIN slack_delivery_state st ON st.open_gap_generation = g.id + WHERE st.id = 1`).Scan(&openedAt, &recoveredAt) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("store: read open delivery gap: %w", err) + } + opened, err := timePtr(openedAt) + if err != nil { + return fmt.Errorf("store: parse open delivery gap opened_at: %w", err) + } + if opened != nil { + age := int64(now.UTC().Sub(*opened).Seconds()) + if age < 0 { + age = 0 + } + stats.OpenGapAgeSeconds = &age + } + recovered, err := timePtr(recoveredAt) + if err != nil { + return fmt.Errorf("store: parse open delivery gap recovered_at: %w", err) + } + if recovered == nil { + return nil + } + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents + WHERE status = 'pending' AND situation_id IS NOT NULL AND created_at <= ?`, + canonicalTime(*recovered)).Scan(&stats.ReplayBacklog); err != nil { + return fmt.Errorf("store: count delivery gap replay backlog: %w", err) + } + return nil +} + +func readUncertainAndBlockedStatsTx(ctx context.Context, tx *sql.Tx, stats *NotificationDeliveryStats) error { + placeholders, args := inPlaceholders(uncertainDeliveryErrorClasses) + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents WHERE last_error_class IN (`+placeholders+`)`, args...). // #nosec G202 -- placeholders is a fixed "?,?,?" run over a package-local constant list; every value is bound + Scan(&stats.UncertainOutcomes); err != nil { + return fmt.Errorf("store: count uncertain delivery outcomes: %w", err) + } + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents WHERE status = 'blocked_configuration'`). + Scan(&stats.BlockedConfigurationCount); err != nil { + return fmt.Errorf("store: count blocked notification intents: %w", err) + } + // A blocked root projection that a newer PENDING root projection for the + // same Situation already replaces can never deliver: the newer one edits + // the same root with strictly newer content, and 0018 forbids retiring + // the blocked row through supersession (only a pending row may become + // superseded). Reactivating it would post yesterday's projection. Task 7 + // accepted that as a permanent resident of the blocked count; reporting + // it separately is what lets an operator see the actionable remainder + // reach zero. + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM notification_intents blocked + WHERE blocked.status = 'blocked_configuration' AND blocked.effect_class = 'root_sync' + AND EXISTS ( + SELECT 1 FROM notification_intents newer + WHERE newer.situation_id = blocked.situation_id + AND newer.effect_class = 'root_sync' AND newer.status = 'pending' + AND newer.summary_version > blocked.summary_version)`). + Scan(&stats.BlockedConfigurationSubsumed); err != nil { + return fmt.Errorf("store: count subsumed blocked notification intents: %w", err) + } + return nil +} From 8521623bd49d37d837511f6186a31938be5876a9 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 08:50:13 +0300 Subject: [PATCH 16/31] fix(runtime): clamp journal pages, surface post-closure artifacts Review fixes for Task 9: - alertint_list_situation_transitions clamps limit to its advertised maximum, so an oversized limit can no longer return a truncated page with next_cursor: null. - alertint_get_situation lists R2's operator artifacts recorded after closure, which no other surface exposed from the Situation. - The stdout stream's retry backoff keys off the row's durable attempt count instead of the Transition sequence. - Startup step 6's stale-root sweep no longer depends on the boot-time Slack probe; only reactivation and gap replay do. Signed-off-by: ernescz --- cmd/alertint/situation_notifications.go | 75 +++++-- cmd/alertint/situation_notifications_test.go | 20 ++ docs/integrations/mcp-clients.md | 2 +- internal/mcp/server_situations.go | 89 ++++++-- internal/mcp/server_situations_test.go | 195 ++++++++++++++++++ internal/notify/stdout/situation.go | 5 +- internal/notify/stdout/situation_test.go | 66 +++++- internal/situation/telemetry_test.go | 40 +++- internal/store/situation_history.go | 10 +- internal/store/situation_transition_stream.go | 12 +- .../store/situation_transition_stream_test.go | 87 ++++++++ internal/store/situation_views.go | 84 ++++++++ 12 files changed, 630 insertions(+), 55 deletions(-) diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go index 42aa564..f7900eb 100644 --- a/cmd/alertint/situation_notifications.go +++ b/cmd/alertint/situation_notifications.go @@ -550,6 +550,11 @@ type notificationRecovery struct { // 5. reactivate eligible configuration-blocked intents; and // 6. perform stale-root supersession and resume ordered recovery replay. // +// Step 6's supersession half runs unconditionally; step 5 and step 6's replay +// half are the only two pieces that require the Slack probe to have +// succeeded, because only a corrected configuration may reactivate a blocked +// intent and only a live Slack may move a gap generation into replay. +// // It publishes nothing. Steps 3 and 6 only pull a Situation's next // assessment forward, so the ordinary controller path decides — under the // ordinary materiality and publication rules — whether anything is committed @@ -592,11 +597,35 @@ func (r *notificationRuntime) RecoverAndReactivate(ctx context.Context, now time report.ConfigurationGeneration = state.ConfigurationGeneration report.BlockedConfigurationRetained = state.BlockedConfigurationCount - if !report.SlackConfigurationValid { - return report, nil + // Step 5 — gated: only a CORRECTED Slack configuration may return a + // blocked intent to pending, and a failed probe has not corrected + // anything. The worker's own first successful probe applies it later if + // Slack comes back within this process's lifetime. + if report.SlackConfigurationValid { + if err := r.reactivateConfiguration(ctx, &report); err != nil { + return report, err + } } - if err := r.reactivateAndResume(ctx, now, &report); err != nil { - return report, err + + // Step 6a — UNCONDITIONAL. Stale-root supersession scheduling is + // publication-free and touches only durable root-supersession state, so + // it needs no reachable Slack at all. It is also startup-only with no + // steady-state equivalent, so gating it on the probe would let one + // transient boot-time Slack blip skip it for the whole process lifetime. + staleRoots, err := r.store.ScheduleSituationsWithStaleRootProjection(ctx, now) + if err != nil { + return report, fmt.Errorf("situation notifications: schedule situations with a stale root projection: %w", err) + } + report.ScheduledStaleRoot = staleRoots + + // Step 6b — gated: a gap generation may only move to replaying once + // Slack actually answers again. Unlike step 6a this HAS a steady-state + // equivalent — the worker's probe loop calls RecoverDeliveryGap on every + // successful probe — so skipping it here costs latency, never coverage. + if report.SlackConfigurationValid { + if err := r.resumeGapReplay(ctx, now, &report); err != nil { + return report, err + } } return report, nil } @@ -617,28 +646,30 @@ func (r *notificationRuntime) validateSlackConfiguration(ctx context.Context) bo return true } -// reactivateAndResume is startup steps 5 and 6, reached only once the Slack -// configuration has actually validated. -func (r *notificationRuntime) reactivateAndResume(ctx context.Context, now time.Time, report *notificationRecovery) error { - if r.worker != nil { - // Exactly once per process (the worker's own one-shot guard makes - // this idempotent against its first steady-state probe, whichever - // runs first). - n, err := r.worker.ReactivateConfiguration(ctx) - if err != nil { - return fmt.Errorf("situation notifications: reactivate configuration-blocked intents: %w", err) - } - report.Reactivated = n - if n > 0 && report.BlockedConfigurationRetained >= n { - report.BlockedConfigurationRetained -= n - } +// reactivateConfiguration is startup step 5: return every eligible +// configuration-blocked intent to pending under a fresh configuration +// generation, exactly once per process (the worker's own one-shot guard makes +// this idempotent against its first steady-state probe, whichever runs +// first). A no-Slack build has no worker and nothing to reactivate. +func (r *notificationRuntime) reactivateConfiguration(ctx context.Context, report *notificationRecovery) error { + if r.worker == nil { + return nil } - staleRoots, err := r.store.ScheduleSituationsWithStaleRootProjection(ctx, now) + n, err := r.worker.ReactivateConfiguration(ctx) if err != nil { - return fmt.Errorf("situation notifications: schedule situations with a stale root projection: %w", err) + return fmt.Errorf("situation notifications: reactivate configuration-blocked intents: %w", err) } - report.ScheduledStaleRoot = staleRoots + report.Reactivated = n + if n > 0 && report.BlockedConfigurationRetained >= n { + report.BlockedConfigurationRetained -= n + } + return nil +} +// resumeGapReplay is startup step 6's replay half: move a generation left +// open by the previous process into replaying now that Slack has answered, +// rather than waiting for the worker's first steady-state probe. +func (r *notificationRuntime) resumeGapReplay(ctx context.Context, now time.Time, report *notificationRecovery) error { generation, resumed, err := r.store.RecoverDeliveryGap(ctx, now) if err != nil { return fmt.Errorf("situation notifications: resume delivery gap replay: %w", err) diff --git a/cmd/alertint/situation_notifications_test.go b/cmd/alertint/situation_notifications_test.go index c74b414..62cb8bd 100644 --- a/cmd/alertint/situation_notifications_test.go +++ b/cmd/alertint/situation_notifications_test.go @@ -899,6 +899,23 @@ func TestSituationNotificationRuntimeStartupKeepsBlockedIntentsWhenSlackFails(t t.Error("a gap was recovered while Slack was still unreachable") } } + // Finding #4: the stale-root sweep is publication-free and touches only + // durable root-supersession state, so it must NOT be gated on the probe. + // It is startup-only with no steady-state equivalent, so skipping it on a + // transient boot-time Slack blip would mean it never runs again for this + // whole process lifetime. + sawStaleRootSweep := false + for _, phase := range tr.snapshot() { + if phase == "schedule_stale_root_projections" { + sawStaleRootSweep = true + } + } + if !sawStaleRootSweep { + t.Error("the stale-root sweep was skipped because the Slack probe failed; it needs no Slack at all") + } + if report.ScheduledStaleRoot != 1 { + t.Errorf("stale roots scheduled = %d, want 1 even with Slack unreachable", report.ScheduledStaleRoot) + } } // TestSituationNotificationRuntimeStartupWithoutSlackRetainsDurableWork @@ -925,6 +942,9 @@ func TestSituationNotificationRuntimeStartupWithoutSlackRetainsDurableWork(t *te "recover_transition_stream_claims", "schedule_situations_missing_first_transition", "read_slack_delivery_state", + // Publication-free and Slack-independent: it runs even with Slack + // switched off entirely (finding #4). + "schedule_stale_root_projections", } got := tr.snapshot() if len(got) != len(want) { diff --git a/docs/integrations/mcp-clients.md b/docs/integrations/mcp-clients.md index df141c6..43c34aa 100644 --- a/docs/integrations/mcp-clients.md +++ b/docs/integrations/mcp-clients.md @@ -136,7 +136,7 @@ restart Windsurf and check **Settings → MCP Servers**: | `alertint_get_evidence_pack` | Get the evidence pack and Prometheus metrics for an incident. | | `alertint_verify_audit` | Verify the hash-chained audit log and report any tampering. | | `alertint_list_situations` | List durable Situations — the exact-group lineage that durably owns one or more Incidents — most recently updated first. A bounded summary: lifecycle/attention/scheduling fields and due reasons only, no Assessment or controller detail (use `alertint_get_situation` for that) and no Slack presence. | -| `alertint_get_situation` | Get one Situation by id or public handle: its immutable member Incidents (each with its current Triage decision/phase/attempts/due time/covered digests), the current authoritative Assessment and derivation, current operator contract, material/Assessment-basis hashes, the eligible Sufficient-reason candidate set (`eligible_reasons`: identity, code, catalog/predicate versions, evidence references, deterministic-floor flag), up to 20 bounded sanitized recent Assessment attempts, and controller retry/park state. `assessment`/`operator_contract`/the hash fields render as explicit `null`, and `recent_attempts` and `eligible_reasons` as empty arrays, for a Situation the controller has not reconciled at least once yet — never an error, and never a fabricated placeholder. Also carries `episode` — the current Episode summary read coherently with the exact Transition it was folded from (explicit `null` before any Transition exists) — and `slack_delivery`, the Situation's whole Slack presence: whether a root is durably published, where it lives, and every durable delivery obligation's class, status, priority, retry/supersession reason, and delivered coordinates. | +| `alertint_get_situation` | Get one Situation by id or public handle: its immutable member Incidents (each with its current Triage decision/phase/attempts/due time/covered digests), the current authoritative Assessment and derivation, current operator contract, material/Assessment-basis hashes, the eligible Sufficient-reason candidate set (`eligible_reasons`: identity, code, catalog/predicate versions, evidence references, deterministic-floor flag), up to 20 bounded sanitized recent Assessment attempts, and controller retry/park state. `assessment`/`operator_contract`/the hash fields render as explicit `null`, and `recent_attempts` and `eligible_reasons` as empty arrays, for a Situation the controller has not reconciled at least once yet — never an error, and never a fabricated placeholder. Also carries `episode` — the current Episode summary read coherently with the exact Transition it was folded from (explicit `null` before any Transition exists) — and `slack_delivery`, the Situation's whole Slack presence: whether a root is durably published, where it lives, and every durable delivery obligation's class, status, priority, retry/supersession reason, and delivered coordinates. `artifacts_recorded_after_closure` lists operator annotations and Captured verdicts that reached the Situation after it had already closed — recorded against it, never journaled (a closed episode is immutable), and never lost. | | `alertint_list_situation_transitions` | Page one Situation's immutable Transition journal, oldest first. Each Transition is one authoritative material change: lifecycle/attention, operator contract, transition reason, journal kind and bounded journal entry, evidence references, actor, and drill marker. History is never reconstructed from current state — a Situation with no Transition yet returns an empty array. Page with the returned `next_cursor`, a stable `(sequence, id)` position rather than an offset. | | `alertint_get_delivery_state` | Get the installation-level Situation Slack delivery state: the continuous-failure window, the durable Slack configuration generation and how many effects are blocked on it, the current Delivery-gap generation with its status/age and replay backlog, retries and outcomes by effect class, how many outcomes Slack never confirmed either way, and the stdout Transition-stream backlog. Bounded counts and closed codes only — never a token, a Slack response, or a provider error body. | | `prometheus_query` | Instant PromQL query against the connected Prometheus (requires Prometheus enabled). | diff --git a/internal/mcp/server_situations.go b/internal/mcp/server_situations.go index db7b817..9bb6ec7 100644 --- a/internal/mcp/server_situations.go +++ b/internal/mcp/server_situations.go @@ -287,16 +287,19 @@ func (s *Server) handleGetSituation(ctx context.Context, req mcplib.CallToolRequ // never reconstructed from current state — and slack_delivery always // answers, saying "published: false" with no effects for a Situation that // warranted no Slack at all. - episode, delivery, err := s.situationHistoryFor(ctx, sit.ID) + history, err := s.situationHistoryFor(ctx, sit.ID) if err != nil { return errResult("failed to get situation history"), nil } - if episode == nil { + if history.Episode == nil { payload["episode"] = nil } else { - payload["episode"] = episode + payload["episode"] = history.Episode } - payload["slack_delivery"] = delivery + payload["slack_delivery"] = history.Delivery + // R2: recorded after closure, never journaled, never lost. An empty + // array, never null — "none" is an answer, not an absence. + payload["artifacts_recorded_after_closure"] = history.Artifacts result, err := mcplib.NewToolResultJSON(payload) if err != nil { @@ -313,10 +316,13 @@ func (s *Server) handleGetSituation(ctx context.Context, req mcplib.CallToolRequ // join, never a reconstruction of history from current state: // // - alertint_get_situation gains "episode" (the current Episode summary -// read in one snapshot with the exact Transition it was folded from) and +// read in one snapshot with the exact Transition it was folded from), // "slack_delivery" (the current root coordinates plus every durable // effect's status, including the withheld/superseded/delayed decisions -// that are durable rows rather than absent ones); +// that are durable rows rather than absent ones), and +// "artifacts_recorded_after_closure" (R2's operator artifacts that +// reached an already-terminal owner: recorded, never journaled, and +// otherwise invisible from the Situation they name); // - alertint_list_situation_transitions pages the immutable Transition // journal by a stable (sequence, id) cursor; and // - alertint_get_delivery_state exposes the installation-level Slack @@ -328,6 +334,18 @@ func (s *Server) handleGetSituation(ctx context.Context, req mcplib.CallToolRequ // reason every failure path returns a fixed generic message. // ---------------------------------------------------------------------- +// maxSituationTransitionPage mirrors internal/store's own +// maxSituationHistoryPage: the largest Transition page this tool will ever +// return, and the value an oversized `limit` is clamped to. It is stated +// here as well as in the store because the handler's own cursor gate +// compares against the limit it actually used — the two must agree or a +// clamped page would advertise itself as the last one. +// defaultSituationTransitionPage is what an absent or nonsensical limit gets. +const ( + maxSituationTransitionPage = 100 + defaultSituationTransitionPage = 50 +) + func (s *Server) toolListSituationTransitions() (mcplib.Tool, mcpserver.ToolHandlerFunc) { tool := mcplib.NewTool("alertint_list_situation_transitions", mcplib.WithDescription("Page one Situation's immutable Transition journal, oldest first. A Transition "+ @@ -506,12 +524,16 @@ type situationDeliveryRow struct { // state through Task 5's bounded readers. A Situation with no Transition yet // legitimately has no episode at all: that renders as an explicit null, never // as history reconstructed from current state. -func (s *Server) situationHistoryFor(ctx context.Context, situationID string) (*situationEpisodeRow, situationDeliveryRow, error) { - delivery := situationDeliveryRow{Effects: []situationEffectRow{}} +func (s *Server) situationHistoryFor(ctx context.Context, situationID string) (situationHistoryView, error) { + view := situationHistoryView{ + Delivery: situationDeliveryRow{Effects: []situationEffectRow{}}, + Artifacts: []store.OwnerTerminalArtifact{}, + } + delivery := &view.Delivery channel, messageTS, published, err := s.st.GetSituationRootCoordinates(ctx, situationID) if err != nil { - return nil, delivery, err + return view, err } delivery.Published = published if published { @@ -520,21 +542,46 @@ func (s *Server) situationHistoryFor(ctx context.Context, situationID string) (* } intents, err := s.st.ListSituationNotificationIntents(ctx, situationID, 0) if err != nil { - return nil, delivery, err + return view, err } for _, intent := range intents { delivery.Effects = append(delivery.Effects, situationEffectRowFrom(intent)) } - view, err := s.st.GetSituationEpisodeView(ctx, situationID) + // R2: operator artifacts that reached this Situation after it had already + // terminalized. They are recorded, never journaled — the terminal Episode + // is immutable — so no Transition names them and they appear in no + // journal page. Without this list they would exist durably and be + // invisible from the Situation they were written against. + artifacts, err := s.st.ListOwnerTerminalArtifacts(ctx, situationID, 0) + if err != nil { + return view, err + } + if len(artifacts) > 0 { + view.Artifacts = artifacts + } + + episodeView, err := s.st.GetSituationEpisodeView(ctx, situationID) if errors.Is(err, store.ErrNotFound) { - return nil, delivery, nil + return view, nil } if err != nil { - return nil, delivery, err + return view, err } - episode := situationEpisodeRowFrom(view) - return &episode, delivery, nil + episode := situationEpisodeRowFrom(episodeView) + view.Episode = &episode + return view, nil +} + +// situationHistoryView bundles the three bounded history reads +// alertint_get_situation renders: the current Episode projection (nil when +// the Situation has no Transition yet — history is never reconstructed from +// current state), the Situation's whole Slack presence, and R2's +// after-closure operator artifacts. +type situationHistoryView struct { + Episode *situationEpisodeRow + Delivery situationDeliveryRow + Artifacts []store.OwnerTerminalArtifact } // transitionCursorRow is the stable page position a caller resumes from. @@ -548,9 +595,17 @@ func (s *Server) handleListSituationTransitions(ctx context.Context, req mcplib. if failed != nil { return failed, nil } - limit := mcplib.ParseInt(req, "limit", 50) + // BOTH bounds are clamped here, not only the lower one. The Store's own + // ListSituationTransitions clamps internally too, so an unclamped + // oversized limit would come back short while this handler's + // `len(rows) == limit` cursor gate went false — handing the caller a + // truncated journal with next_cursor: null, which reads as "complete". + limit := mcplib.ParseInt(req, "limit", defaultSituationTransitionPage) if limit < 1 { - limit = 50 + limit = defaultSituationTransitionPage + } + if limit > maxSituationTransitionPage { + limit = maxSituationTransitionPage } cursor := store.TransitionCursor{ Sequence: mcplib.ParseInt(req, "cursor_sequence", 0), diff --git a/internal/mcp/server_situations_test.go b/internal/mcp/server_situations_test.go index 2b37aa5..49cb150 100644 --- a/internal/mcp/server_situations_test.go +++ b/internal/mcp/server_situations_test.go @@ -175,6 +175,9 @@ func TestGetSituationByIDExactContract(t *testing.T) { // coherently with its source Transition (explicit null before any // Transition exists) and the Situation's whole Slack presence. "episode", "slack_delivery", + // R2: recorded after closure, never journaled — an empty array here, + // never null, for a Situation nothing was written against post-hoc. + "artifacts_recorded_after_closure", } if len(payload) != len(wantKeys) { t.Fatalf("payload has %d keys, want exactly %d: %+v", len(payload), len(wantKeys), payload) @@ -632,6 +635,34 @@ func TestSituationMCPTransitionJournalPagesByStableCursor(t *testing.T) { t.Fatalf("next_cursor = %+v, want the last row's stable position", page.NextCursor) } + // An over-large limit is clamped to the tool's advertised maximum, and — + // the part that matters — next_cursor must still signal that more data + // remains. Before this was clamped here, the Store's own internal clamp + // silently capped the page while the handler's `len(rows) == limit` gate + // went false, so an immutable journal LOOKED complete when it was not. + over, err := s.handleListSituationTransitions(context.Background(), + reqWith(map[string]any{"id": situationID, "limit": 1000})) + if err != nil || over.IsError { + t.Fatalf("over-large limit errored: %v %s", err, resultText(t, over)) + } + var overPage struct { + Transitions []struct { + Sequence int `json:"sequence"` + } `json:"transitions"` + NextCursor *struct { + Sequence int `json:"sequence"` + } `json:"next_cursor"` + } + if err := json.Unmarshal([]byte(resultText(t, over)), &overPage); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if len(overPage.Transitions) != 5 { + t.Fatalf("over-large limit returned %d transitions, want all 5", len(overPage.Transitions)) + } + if overPage.NextCursor != nil { + t.Fatalf("next_cursor = %+v when the whole ledger fits, want null", overPage.NextCursor) + } + res2, err := s.handleListSituationTransitions(context.Background(), reqWith(map[string]any{ "id": situationID, "limit": 10, "cursor_sequence": page.NextCursor.Sequence, "cursor_id": page.NextCursor.ID, @@ -656,6 +687,71 @@ func TestSituationMCPTransitionJournalPagesByStableCursor(t *testing.T) { } } +// TestSituationMCPTransitionJournalClampsAnOverLargeLimitAndStillPages is the +// finding-#1 regression: with MORE rows than the clamp, an over-large limit +// must return exactly the clamped page AND a next_cursor, so a caller can +// never mistake a truncated page for a complete journal. +func TestSituationMCPTransitionJournalClampsAnOverLargeLimitAndStillPages(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-clamp", "service=clamp", "incident_created", at) + seedSituationHistoryForMCP(t, st, situationID, at, maxSituationTransitionPage+5) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleListSituationTransitions(context.Background(), + reqWith(map[string]any{"id": situationID, "limit": 1000})) + if err != nil || res.IsError { + t.Fatalf("list transitions errored: %v %s", err, resultText(t, res)) + } + var page struct { + Transitions []struct { + ID string `json:"id"` + Sequence int `json:"sequence"` + } `json:"transitions"` + NextCursor *struct { + Sequence int `json:"sequence"` + ID string `json:"id"` + } `json:"next_cursor"` + } + if err := json.Unmarshal([]byte(resultText(t, res)), &page); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if len(page.Transitions) != maxSituationTransitionPage { + t.Fatalf("limit 1000 returned %d transitions, want the clamped %d", + len(page.Transitions), maxSituationTransitionPage) + } + if page.NextCursor == nil { + t.Fatal("next_cursor is null on a clamped page; the caller cannot tell the journal is incomplete") + } + if page.NextCursor.Sequence != maxSituationTransitionPage { + t.Errorf("next_cursor sequence = %d, want %d", page.NextCursor.Sequence, maxSituationTransitionPage) + } + + // Resuming from it returns the remainder and then stops. + rest, err := s.handleListSituationTransitions(context.Background(), reqWith(map[string]any{ + "id": situationID, "limit": 1000, + "cursor_sequence": page.NextCursor.Sequence, "cursor_id": page.NextCursor.ID, + })) + if err != nil || rest.IsError { + t.Fatalf("second page errored: %v %s", err, resultText(t, rest)) + } + var restPage struct { + Transitions []struct { + Sequence int `json:"sequence"` + } `json:"transitions"` + NextCursor *struct{} `json:"next_cursor"` + } + if err := json.Unmarshal([]byte(resultText(t, rest)), &restPage); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if len(restPage.Transitions) != 5 { + t.Fatalf("remainder = %d transitions, want 5", len(restPage.Transitions)) + } + if restPage.NextCursor != nil { + t.Error("next_cursor is non-null on the final page") + } +} + // TestSituationMCPGetSituationCarriesEpisodeAndDeliveryState proves the // extended alertint_get_situation payload: the current Episode summary read // coherently with its own source Transition, the Slack root coordinates and @@ -820,3 +916,102 @@ func TestSituationMCPHistoryIsAbsentNotFabricated(t *testing.T) { t.Errorf("journal for a Situation with no history = %s, want an empty array", resultText(t, res2)) } } + +// seedOwnerTerminalArtifactForMCP writes one operator artifact that reached +// an already-terminal owner (R2): applied, recorded against that owner, and +// deliberately never journaled — `journal_state = 'owner_terminal'`. +func seedOwnerTerminalArtifactForMCP(t *testing.T, st *store.Store, situationID, incidentID string, at time.Time) { + t.Helper() + ctx := context.Background() + res, err := st.DB().ExecContext(ctx, ` + INSERT INTO incident_annotations (incident_id, kind, note, created_at) + VALUES (?, 'observation', 'operator note recorded after closure', ?)`, + incidentID, at.UTC().Format(time.RFC3339Nano)) + if err != nil { + t.Fatalf("insert annotation: %v", err) + } + annotationID, err := res.LastInsertId() + if err != nil { + t.Fatalf("annotation id: %v", err) + } + id := "input-owner-terminal-" + incidentID + if _, err := st.DB().ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, status, + applied_situation_id, applied_at, applied_input_version, annotation_id, journal_state + ) VALUES (?, ?, ?, 'operator_annotation_recorded', 'service=closed', ?, 'applied', ?, ?, 1, ?, 'owner_terminal')`, + id, "idem:"+id, incidentID, at.UTC().Format(time.RFC3339Nano), + situationID, at.UTC().Format(time.RFC3339Nano), annotationID); err != nil { + t.Fatalf("insert owner_terminal outbox row: %v", err) + } +} + +// TestSituationMCPListsArtifactsRecordedAfterClosure is R2's completion-gate +// requirement: an operator artifact applied after its Situation closed is +// recorded as owner_terminal, never journaled, and must remain VISIBLE in the +// Situation MCP view — not silently lost between the Incident surfaces and +// the immutable Transition journal it was deliberately kept out of. +func TestSituationMCPListsArtifactsRecordedAfterClosure(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-closed", "service=closed", "incident_created", at) + seedSituationHistoryForMCP(t, st, situationID, at, 1) + seedOwnerTerminalArtifactForMCP(t, st, situationID, "inc-closed", at.Add(time.Hour)) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleGetSituation(context.Background(), reqWith(map[string]any{"id": situationID})) + if err != nil || res.IsError { + t.Fatalf("get situation errored: %v %s", err, resultText(t, res)) + } + raw := resultText(t, res) + var payload map[string]any + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + artifacts, ok := payload["artifacts_recorded_after_closure"].([]any) + if !ok { + t.Fatalf("payload has no artifacts_recorded_after_closure array: %v", payload["artifacts_recorded_after_closure"]) + } + if len(artifacts) != 1 { + t.Fatalf("artifacts_recorded_after_closure = %v, want exactly the one owner_terminal artifact", artifacts) + } + row, _ := artifacts[0].(map[string]any) + if row["kind"] != "operator_annotation_recorded" { + t.Errorf("artifact kind = %v", row["kind"]) + } + if row["incident_id"] != "inc-closed" { + t.Errorf("artifact incident_id = %v, want inc-closed", row["incident_id"]) + } + if row["annotation_id"] == nil { + t.Error("artifact carries no annotation id; the operator cannot follow it to the Incident surface") + } + if row["occurred_at"] == nil || row["applied_at"] == nil { + t.Errorf("artifact lost its instants: %v", row) + } + // It is recorded, never journaled: no Transition names it. + if _, ok := row["journaled_transition_id"]; ok { + t.Error("an owner_terminal artifact must never claim a journaling Transition") + } + for _, forbidden := range []string{"lease_owner", "claim_token", "SELECT "} { + if strings.Contains(raw, forbidden) { + t.Errorf("payload leaks %q", forbidden) + } + } +} + +// TestSituationMCPArtifactsAfterClosureIsEmptyWhenNoneExist proves the +// ordinary case renders as an empty array, never null and never an error. +func TestSituationMCPArtifactsAfterClosureIsEmptyWhenNoneExist(t *testing.T) { + st := newMCPStore(t) + at := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC) + situationID := seedSituationForMCP(t, st, "inc-open", "service=open", "incident_created", at) + + s := NewServer(Config{}, st, audit.New(st.DB())) + res, err := s.handleGetSituation(context.Background(), reqWith(map[string]any{"id": situationID})) + if err != nil || res.IsError { + t.Fatalf("get situation errored: %v %s", err, resultText(t, res)) + } + if !strings.Contains(resultText(t, res), `"artifacts_recorded_after_closure":[]`) { + t.Errorf("payload = %s, want an empty artifacts_recorded_after_closure array", resultText(t, res)) + } +} diff --git a/internal/notify/stdout/situation.go b/internal/notify/stdout/situation.go index 95da3a3..37d764e 100644 --- a/internal/notify/stdout/situation.go +++ b/internal/notify/stdout/situation.go @@ -397,7 +397,10 @@ func (w *TransitionStreamWorker) acknowledge(ctx context.Context, claim store.Tr w.audit(ctx, AuditTransitionStreamFailed, claim, streamErrorInvalid) return nil default: - delay := streamRetryDelay(claim.Transition.Sequence, w.cfg.RetryInitial, w.cfg.RetryMax) + // The row's own durable attempt count, never the Transition's + // sequence: sequence is an ordering position with no relationship to + // how many times THIS row has failed. + delay := streamRetryDelay(claim.AttemptCount, w.cfg.RetryInitial, w.cfg.RetryMax) if err := w.store.RetryTransitionStreamEntry(ctx, claim, streamErrorUnavailable, now.Add(delay)); err != nil { w.noteClaimLoss(err) return err diff --git a/internal/notify/stdout/situation_test.go b/internal/notify/stdout/situation_test.go index f3b2f8f..3edd723 100644 --- a/internal/notify/stdout/situation_test.go +++ b/internal/notify/stdout/situation_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "log/slog" "strings" "sync" "testing" @@ -142,6 +143,10 @@ func (f *tsFakeStore) ClaimTransitionStream(_ context.Context, owner string, now r.leased = true r.claim.ClaimOwner = owner r.claim.ClaimToken++ + // The real store increments attempt_count in the same claiming + // UPDATE; the fake must too, or the retry schedule under test would + // never advance. + r.claim.AttemptCount++ out = append(out, r.claim) } return out, nil @@ -283,7 +288,8 @@ func (a *tsFakeAuditor) kinds() []string { func tsWorker(w *tsFailingWriter, st TransitionStreamStore, auditor TransitionStreamAuditSink) *TransitionStreamWorker { clock := func() time.Time { return time.Date(2026, 9, 5, 12, 0, 0, 0, time.UTC) } - return NewTransitionStreamWorker(w, st, TransitionStreamConfig{Owner: "test-owner"}, auditor, clock, nil) + return NewTransitionStreamWorker(w, st, TransitionStreamConfig{Owner: "test-owner"}, auditor, clock, + slog.New(slog.DiscardHandler)) } func tsLines(t *testing.T, out string) []map[string]any { @@ -647,3 +653,61 @@ func TestSituationTransitionStreamEmitSpanUsesTheSituationScope(t *testing.T) { } } } + +// TestSituationTransitionStreamBacksOffByAttemptNotSequence pins the fix for +// review finding #3: the retry schedule is keyed off the row's own durable +// attempt count, exactly like the sibling notification worker's, NOT off the +// Transition's sequence number. +// +// Keying it off sequence produced two opposite bugs at once: a Situation's +// FIRST Transition (sequence 1) retried at the initial delay forever — a hot +// loop against an unwritable stdout — while a Transition at sequence 6 or +// beyond jumped straight to the cap on its very first failure, never trying +// the fast retries that recover from a momentary blip. +func TestSituationTransitionStreamBacksOffByAttemptNotSequence(t *testing.T) { + // A high-sequence Transition on its FIRST attempt must use the INITIAL + // delay, not the cap. + high := tsTransition(t, 9, model.LifecycleActive, model.ReasonAttentionChanged) + st := newTSFakeStore(high) + w := &tsFailingWriter{fail: true} + worker := tsWorker(w, st, nil) + if _, err := worker.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + row := st.snapshot()[0] + if row.retryAt == nil { + t.Fatal("no retry scheduled") + } + firstDelay := row.retryAt.Sub(worker.now()) + if firstDelay != defaultStreamRetryInitial { + t.Fatalf("first attempt on a sequence-9 Transition scheduled %v out, want the initial %v "+ + "(the schedule must key off attempt count, not sequence)", firstDelay, defaultStreamRetryInitial) + } + + // A LOW-sequence Transition must still back off as its attempts pile up, + // rather than hammering at the initial delay forever. + low := tsTransition(t, 1, model.LifecycleActive, model.ReasonFirstAuthoritativeState) + st2 := newTSFakeStore(low) + w2 := &tsFailingWriter{fail: true} + worker2 := tsWorker(w2, st2, nil) + delays := make([]time.Duration, 0, 4) + for range 4 { + st2.rows[0].retryAt = nil + if _, err := worker2.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + r := st2.snapshot()[0] + if r.retryAt == nil { + t.Fatal("no retry scheduled") + } + delays = append(delays, r.retryAt.Sub(worker2.now())) + } + for i := 1; i < len(delays); i++ { + if delays[i] <= delays[i-1] { + t.Fatalf("delays = %v; a sequence-1 Transition must back off across attempts, not retry at a fixed interval", delays) + } + } + if delays[0] != defaultStreamRetryInitial { + t.Errorf("first delay = %v, want the initial %v", delays[0], defaultStreamRetryInitial) + } +} diff --git a/internal/situation/telemetry_test.go b/internal/situation/telemetry_test.go index 308559e..cbfbbc6 100644 --- a/internal/situation/telemetry_test.go +++ b/internal/situation/telemetry_test.go @@ -374,21 +374,47 @@ func TestTelemetryAuditKindsMatchTheAuditCatalog(t *testing.T) { t.Errorf("internal/situation emits %q, which internal/audit's catalog does not name", kind) } } - // The two the stdout stream worker owns complete the catalog; nothing - // else may be left unclaimed. + // Every catalog name must be claimed by some emitter — except the ones + // explicitly listed below as knowingly unemitted, which is the ONLY way + // this assertion stays honest: silently folding an unemitted name into + // the claimed set would make "no catalog name is unclaimed" unable to + // catch the very thing it exists to catch. claimed := map[string]bool{ + // Emitted by internal/notify/stdout's TransitionStreamWorker. audit.KindTransitionStreamEmitted: true, audit.KindTransitionStreamFailed: true, - // R2's owner_terminal event belongs to the input-application path - // (internal/store), which records the artifact without journaling it. - audit.KindHistoryArtifactOwnerTerminal: true, } for _, kind := range situation.AuditKindsEmittedHere() { claimed[kind] = true } + // RESERVED, NOT YET EMITTED. R2's owner-terminal outcome is decided inside + // Store.ApplySituationInput, which returns only an error — no caller can + // tell that outcome from an ordinary attach, and neither the store nor + // the input worker has an audit seam. Wiring it requires widening that + // function's return contract (Plan 3's input-application boundary, closed + // and reviewed in an earlier task), so it is a tracked follow-up. The + // name is catalogued because spec.md's event list requires it; it is + // listed HERE, separately and by name, so this test says out loud that it + // has no emitter rather than pretending it has one. + knownUnemitted := map[string]string{ + audit.KindHistoryArtifactOwnerTerminal: "follow-up: needs Store.ApplySituationInput to report its R2 outcome", + } for kind := range catalog { - if !claimed[kind] { - t.Errorf("catalog names %q but no emitter claims it", kind) + if claimed[kind] { + continue + } + if why, ok := knownUnemitted[kind]; ok { + t.Logf("catalog name %q is reserved with no emitter (%s)", kind, why) + continue + } + t.Errorf("catalog names %q but no emitter claims it", kind) + } + // The reserved list must stay a list of genuinely unemitted names: if an + // emitter ever appears for one, this fails so the entry is removed rather + // than left as a stale excuse. + for kind := range knownUnemitted { + if claimed[kind] { + t.Errorf("%q is listed as unemitted but an emitter now claims it; drop it from knownUnemitted", kind) } } } diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go index c7fd815..b58bf40 100644 --- a/internal/store/situation_history.go +++ b/internal/store/situation_history.go @@ -682,10 +682,12 @@ func scanStreamEntry(rows *sql.Rows, streamID *string) (situationmodel.Transitio } // scanClaimedStreamEntry scans one -// `SELECT st.id, st.claim_token, ` row: the -// stream row's own id and current claim token, then the joined Transition. -func scanClaimedStreamEntry(rows *sql.Rows, streamID *string, claimToken *int64) (situationmodel.Transition, error) { - tr, err := scanTransition(prefixedScanner{rows: rows, prefix: []any{streamID, claimToken}}) +// `SELECT st.id, st.claim_token, st.attempt_count, ` +// row: the stream row's own id, current claim token, and durable attempt +// count, then the joined Transition. +func scanClaimedStreamEntry(rows *sql.Rows, streamID *string, claimToken *int64, + attemptCount *int) (situationmodel.Transition, error) { + tr, err := scanTransition(prefixedScanner{rows: rows, prefix: []any{streamID, claimToken, attemptCount}}) if err != nil { return situationmodel.Transition{}, fmt.Errorf("store: scan claimed transition stream entry: %w", err) } diff --git a/internal/store/situation_transition_stream.go b/internal/store/situation_transition_stream.go index 8a53242..416052b 100644 --- a/internal/store/situation_transition_stream.go +++ b/internal/store/situation_transition_stream.go @@ -45,6 +45,14 @@ type TransitionStreamClaim struct { Transition situationmodel.Transition ClaimOwner string ClaimToken int64 + // AttemptCount is the row's durable attempt count INCLUDING this claim + // (claiming increments it). It is what a worker's retry schedule must + // key off — the same role situation.NotificationClaim.Intent.AttemptCount + // plays for the notification worker. Keying a backoff off the + // Transition's sequence instead would make a Situation's first + // Transition retry at the initial delay forever while a later one + // started at the cap. + AttemptCount int } // ClaimTransitionStream leases up to limit due, pending stream rows in one @@ -121,7 +129,7 @@ func (s *Store) ClaimTransitionStream(ctx context.Context, owner string, now tim func loadClaimedTransitionStreamTx(ctx context.Context, tx *sql.Tx, ids []string, owner string) ([]TransitionStreamClaim, error) { placeholders, args := inPlaceholders(ids) rows, err := tx.QueryContext(ctx, ` - SELECT st.id, st.claim_token, `+prefixedTransitionColumns+` + SELECT st.id, st.claim_token, st.attempt_count, `+prefixedTransitionColumns+` FROM situation_transition_stream st JOIN situation_transitions t ON t.id = st.transition_id WHERE st.id IN (`+placeholders+`) @@ -134,7 +142,7 @@ func loadClaimedTransitionStreamTx(ctx context.Context, tx *sql.Tx, ids []string out := make([]TransitionStreamClaim, 0, len(ids)) for rows.Next() { claim := TransitionStreamClaim{ClaimOwner: owner} - tr, err := scanClaimedStreamEntry(rows, &claim.StreamID, &claim.ClaimToken) + tr, err := scanClaimedStreamEntry(rows, &claim.StreamID, &claim.ClaimToken, &claim.AttemptCount) if err != nil { return nil, err } diff --git a/internal/store/situation_transition_stream_test.go b/internal/store/situation_transition_stream_test.go index 0d9cf8c..4a53c1c 100644 --- a/internal/store/situation_transition_stream_test.go +++ b/internal/store/situation_transition_stream_test.go @@ -74,6 +74,11 @@ func TestTransitionStreamClaimJoinsItsTransitionAndFencesAcknowledgement(t *test if c.ClaimOwner != "stdout-a" || c.ClaimToken < 1 { t.Errorf("claim %d fencing pair = (%q, %d)", i, c.ClaimOwner, c.ClaimToken) } + // The claim carries the row's own durable attempt count (claiming + // increments it), which is what a worker's retry schedule keys off. + if c.AttemptCount != 1 { + t.Errorf("claim %d attempt count = %d, want 1 after the first claim", i, c.AttemptCount) + } } // A held lease is not re-claimable until it expires. @@ -146,6 +151,9 @@ func TestTransitionStreamRetryFailAndRecoverExpiredClaims(t *testing.T) { if err != nil || len(due) != 1 { t.Fatalf("claim after retry_at = %d rows, %v; want 1", len(due), err) } + if due[0].AttemptCount != 2 { + t.Errorf("attempt count on the re-claim = %d, want 2 (it advances so the backoff does too)", due[0].AttemptCount) + } // An abandoned lease is swept without changing status or attempt count. var attemptsBefore int @@ -378,3 +386,82 @@ func TestNotificationDeliveryStatsAreBoundedCounts(t *testing.T) { t.Errorf("first intent = %q, want the root projection first", intents[0].EffectClass) } } + +// TestListOwnerTerminalArtifactsReadsOnlyRecordedNeverJournaledOnes is R2's +// read half: an artifact applied to an already-terminal owner is recorded +// with journal_state='owner_terminal' and must surface here, while a pending +// or journaled artifact — which the Transition journal already carries — +// must not. +func TestListOwnerTerminalArtifactsReadsOnlyRecordedNeverJournaledOnes(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID := newSituationForGroup(t, st, "service=closed-owner", now) + + var incidentID string + if err := st.db.QueryRowContext(ctx, + `SELECT incident_id FROM situation_incidents WHERE situation_id = ?`, sitID).Scan(&incidentID); err != nil { + t.Fatalf("read member incident: %v", err) + } + res, err := st.db.ExecContext(ctx, ` + INSERT INTO incident_annotations (incident_id, kind, note, created_at) + VALUES (?, 'observation', 'recorded after closure', ?)`, + incidentID, canonicalTime(now)) + if err != nil { + t.Fatalf("insert annotation: %v", err) + } + annotationID, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + + insert := func(id, journalState string, at time.Time) { + t.Helper() + if _, err := st.db.ExecContext(ctx, ` + INSERT INTO situation_input_outbox ( + id, idempotency_key, incident_id, kind, group_key, occurred_at, status, + applied_situation_id, applied_at, applied_input_version, annotation_id, journal_state + ) VALUES (?, ?, ?, 'operator_annotation_recorded', 'service=closed-owner', ?, 'applied', ?, ?, 1, ?, ?)`, + id, "idem:"+id, incidentID, canonicalTime(at), sitID, canonicalTime(at), annotationID, journalState); err != nil { + t.Fatalf("insert %s outbox row: %v", journalState, err) + } + } + insert("in-terminal-b", "owner_terminal", now.Add(2*time.Minute)) + insert("in-terminal-a", "owner_terminal", now.Add(time.Minute)) + insert("in-pending", "pending", now.Add(3*time.Minute)) + + artifacts, err := st.ListOwnerTerminalArtifacts(ctx, sitID, 0) + if err != nil { + t.Fatalf("ListOwnerTerminalArtifacts: %v", err) + } + if len(artifacts) != 2 { + t.Fatalf("artifacts = %+v, want exactly the two owner_terminal rows", artifacts) + } + if artifacts[0].InputID != "in-terminal-a" || artifacts[1].InputID != "in-terminal-b" { + t.Fatalf("order = %q,%q; want oldest first", artifacts[0].InputID, artifacts[1].InputID) + } + a := artifacts[0] + if a.Kind != "operator_annotation_recorded" || a.IncidentID != incidentID { + t.Errorf("artifact provenance = %+v", a) + } + if a.AnnotationID == nil || *a.AnnotationID != annotationID { + t.Errorf("annotation id = %v, want %d", a.AnnotationID, annotationID) + } + if a.VerdictID != nil { + t.Errorf("verdict id = %v on an annotation artifact", *a.VerdictID) + } + if a.OccurredAt.IsZero() || a.AppliedAt.IsZero() { + t.Errorf("artifact lost its instants: %+v", a) + } + + // A Situation nothing was recorded against post-hoc reads as an empty + // slice, never nil — "none" is an answer. + other := newSituationForGroup(t, st, "service=untouched", now) + empty, err := st.ListOwnerTerminalArtifacts(ctx, other, 0) + if err != nil { + t.Fatalf("ListOwnerTerminalArtifacts (empty): %v", err) + } + if empty == nil || len(empty) != 0 { + t.Fatalf("artifacts for an untouched Situation = %v, want an empty slice", empty) + } +} diff --git a/internal/store/situation_views.go b/internal/store/situation_views.go index 145bfed..133047b 100644 --- a/internal/store/situation_views.go +++ b/internal/store/situation_views.go @@ -773,3 +773,87 @@ func readUncertainAndBlockedStatsTx(ctx context.Context, tx *sql.Tx, stats *Noti } return nil } + +// OwnerTerminalArtifact is one operator artifact that reached an +// already-terminal Situation owner (R2): applied and recorded against that +// owner, deliberately never journaled, and never lost. +// +// It carries provenance and instants only — the artifact's own content +// (annotation note, verdict expectation) stays where it already lives, on +// the Incident surfaces, which AnnotationID/VerdictID point at. +type OwnerTerminalArtifact struct { + InputID string `json:"input_id"` + Kind string `json:"kind"` + IncidentID string `json:"incident_id"` + AnnotationID *int64 `json:"annotation_id"` + VerdictID *int64 `json:"verdict_id"` + OccurredAt time.Time `json:"occurred_at"` + AppliedAt time.Time `json:"applied_at"` + AppliedInputVersion *int `json:"applied_input_version"` +} + +// ListOwnerTerminalArtifacts reads the operator artifacts recorded against +// situationID after it had already terminalized (R2's `journal_state = +// 'owner_terminal'`), oldest first. limit is clamped to +// maxSituationHistoryPage. +// +// This is the read half of R2's "the artifact stays visible through Incident +// MCP/audit, and the Situation MCP view lists it under 'operator artifacts +// recorded after closure'": the terminal Episode is immutable, so these +// artifacts have no Transition and appear in no journal — without this view +// they would exist durably and be invisible from the Situation they name. +func (s *Store) ListOwnerTerminalArtifacts(ctx context.Context, situationID string, limit int) ([]OwnerTerminalArtifact, error) { + if strings.TrimSpace(situationID) == "" { + return nil, errors.New("store: owner-terminal artifact read requires a situation id") + } + if limit <= 0 || limit > maxSituationHistoryPage { + limit = maxSituationHistoryPage + } + rows, err := s.db.QueryContext(ctx, ` + SELECT id, kind, incident_id, annotation_id, verdict_id, occurred_at, applied_at, applied_input_version + FROM situation_input_outbox + WHERE applied_situation_id = ? AND journal_state = 'owner_terminal' + ORDER BY occurred_at ASC, id ASC + LIMIT ?`, situationID, limit) + if err != nil { + return nil, fmt.Errorf("store: list owner-terminal artifacts: %w", err) + } + defer func() { _ = rows.Close() }() + + out := []OwnerTerminalArtifact{} + for rows.Next() { + var a OwnerTerminalArtifact + var annotationID, verdictID, appliedVersion sql.NullInt64 + var occurredAt, appliedAt string + if err := rows.Scan(&a.InputID, &a.Kind, &a.IncidentID, &annotationID, &verdictID, + &occurredAt, &appliedAt, &appliedVersion); err != nil { + return nil, fmt.Errorf("store: scan owner-terminal artifact: %w", err) + } + if annotationID.Valid { + a.AnnotationID = &annotationID.Int64 + } + if verdictID.Valid { + a.VerdictID = &verdictID.Int64 + } + if appliedVersion.Valid { + v := int(appliedVersion.Int64) + a.AppliedInputVersion = &v + } + for _, f := range []struct { + name string + src string + dst *time.Time + }{{"occurred_at", occurredAt, &a.OccurredAt}, {"applied_at", appliedAt, &a.AppliedAt}} { + parsed, err := time.Parse(time.RFC3339Nano, f.src) + if err != nil { + return nil, fmt.Errorf("store: parse owner-terminal artifact %s: %w", f.name, err) + } + *f.dst = parsed.UTC() + } + out = append(out, a) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("store: iterate owner-terminal artifacts: %w", err) + } + return out, nil +} From 19755af24bee11d2b8bf2d596a78f97a79061e80 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 09:47:20 +0300 Subject: [PATCH 17/31] docs(situation): close history and Slack ownership - internal/situation/history_replay_test.go: real-Store crash-boundary replay for Plan 3 history. Reuses Plan 2's replayFixture, faultyControllerStore, crash points, and simulateCrash rather than a second harness. Eight scripted episodes (first publication, investigation, operator artifacts, artifact after closure, recovery / refire / recovered, closed_unknown, deadline refresh, recurrence lineage + handoff) each run once uninterrupted and once per crash boundary on the same logical-clock schedule, then compare a normalized Transition/Episode/intent/root/gap projection. - cmd/alertint/situation_slack_e2e_test.go: deterministic fake-Slack end-to-end delivery. Real store, controller, notification worker, and SituationDeliverer against a scripted httptest Slack: uncertain post with a lost response, failing root edit, rate limit, invalid token, channel loss, a six-minute outage, recovery probe, ordered replay, a second outage during replay, terminal-before-first-call publication, and stale-handoff demotion. - internal/store: move two #nosec G202 annotations above their statements so gosec honors them (comment-only; gosec -quiet ./... was failing on the branch head). - docs: Slack now documents Situation-owned roots and journals, orientation, durable intents, at-least-once external delivery, indefinite retry, five-minute Delivery gaps, complete replay, configuration blocking, the interruption floor and repage cooldown, stdout deduplication by transition_id, MCP history, the two System exceptions, and no-owner annotation/verdict behaviour. Every claim that an Incident notification path still owns Slack is gone. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- README.md | 12 +- cmd/alertint/situation_slack_e2e_test.go | 1138 +++++++++++++ docs/concepts/architecture.md | 82 +- docs/concepts/scope-and-limits.md | 19 +- docs/getting-started/configuration.md | 27 +- docs/integrations/mcp-clients.md | 12 +- docs/notifications/slack.md | 377 ++++- internal/situation/history_replay_test.go | 1486 +++++++++++++++++ internal/store/situation_notifications.go | 6 +- internal/store/situation_transition_stream.go | 6 +- 10 files changed, 3038 insertions(+), 127 deletions(-) create mode 100644 cmd/alertint/situation_slack_e2e_test.go create mode 100644 internal/situation/history_replay_test.go diff --git a/README.md b/README.md index 7fa4394..fc32695 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ > AlertINT turns infrastructure alerts into investigated incidents and serves them to the AI tools you already use, over MCP — a self-hosted agent that runs inside your own network. -A single Go binary that sits between your monitoring stack and your AI agent. It ingests alert webhooks from Alertmanager and Zabbix, correlates them into incidents through an open rule engine, and runs an LLM triage that falsifies its own draft verdict before the finding ships. Findings go to Slack; the incident state — plus read-only Prometheus, Loki, and Zabbix access — is exposed to any MCP client. Corrections your agent captures over MCP steer the next triage of the same failure. Read-only by design. Local state. You bring the LLM key. +A single Go binary that sits between your monitoring stack and your AI agent. It ingests alert webhooks from Alertmanager and Zabbix, correlates them into incidents through an open rule engine, and runs an LLM triage that falsifies its own draft verdict before the finding ships. Findings go to stdout and, when configured, to one Slack channel; the incident state — plus read-only Prometheus, Loki, and Zabbix access — is exposed to any MCP client. Corrections your agent captures over MCP steer the next triage of the same failure. Read-only by design. Local state. You bring the LLM key. **Full documentation: [alertint.com/docs](https://alertint.com/docs)** @@ -57,6 +57,16 @@ The whole pipeline — receivers, correlation, the evidence pack, both loops, an the MCP surface — is diagrammed and walked through step by step in **[Architecture](https://alertint.com/docs/concepts/architecture)**. +On the `state-controller` integration branch (not the released default), a +durable **Situation** owns each failure group's history: every authoritative +material change commits one immutable transition and one version of a current +episode summary, and Slack shows one evolving Situation root plus an +immutable ordered journal thread instead of a per-incident card. Delivery is +driven from durable intents that retry indefinitely, open a visible gap after +five continuous minutes of Slack failure, and replay every affected episode +in order once Slack returns — see +**[Slack](https://alertint.com/docs/notifications/slack)**. + ## Documentation - **[Docs home](https://alertint.com/docs)** — quickstart, configuration reference diff --git a/cmd/alertint/situation_slack_e2e_test.go b/cmd/alertint/situation_slack_e2e_test.go new file mode 100644 index 0000000..c789ee1 --- /dev/null +++ b/cmd/alertint/situation_slack_e2e_test.go @@ -0,0 +1,1138 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package main + +// Plan 3 Task 10, Step 2: the deterministic fake-Slack end-to-end test. +// +// Everything in the delivery path below is the real, unmodified production +// component: a real *store.Store on disk, the real situation.Controller +// committing real Transitions/Episode summaries/notification intents, the +// real situation.NotificationWorker claiming and acknowledging them, the +// real cmd/alertint SituationDeliverer rendering them, and the real +// internal/notify/slack.Client putting them on the wire. The ONLY fake is +// the Slack Web API itself: an httptest server this file scripts response +// by response, so every failure mode the spec names — an uncertain post +// whose response is lost, a failing root edit, a rate limit, an invalid +// token, a lost channel, a multi-minute outage, recovery, replay, and a +// second outage during that replay — is reproduced deterministically +// against a fake clock, with no live workspace and no network. +// +// The L2 (Situation Assessment) boundary is faked the same way every other +// controller test in this repo fakes it, because this file's subject is +// delivery, not assessment. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/audit" + "github.com/alertint/alertint-agent/internal/llm" + notifyslack "github.com/alertint/alertint-agent/internal/notify/slack" + "github.com/alertint/alertint-agent/internal/situation" + "github.com/alertint/alertint-agent/internal/situation/model" + "github.com/alertint/alertint-agent/internal/store" +) + +// ---------------------------------------------------------------------- +// Scripted fake Slack Web API. +// ---------------------------------------------------------------------- + +// e2eSlackCall is one request the fake provider actually accepted, in +// arrival order. It records exactly the fields the at-least-once and +// ordering assertions need — never a token. +type e2eSlackCall struct { + Method string + ClientMsgID string + Channel string + ThreadTS string + Broadcast bool + TS string + Text string + // Accepted is false when the fake decided to reject or drop this call; + // a dropped call still counts as one the PROVIDER saw, which is the + // whole point of the uncertain-response case. + Accepted bool +} + +// e2eSlackReply is what the fake does with one request. +type e2eSlackReply struct { + // Drop makes the handler hijack and close the connection after + // recording the call: Slack accepted the message, the client never + // learned the outcome. + Drop bool + // ErrorCode returns {"ok":false,"error":code} with HTTP 200, the shape + // Slack itself uses for application-level rejections. + ErrorCode string + // HTTPStatus, when non-zero, returns that status instead of a JSON + // envelope — 429 for a rate limit, 503 for an outage. + HTTPStatus int + // RetryAfterSeconds sets the Retry-After header. + RetryAfterSeconds int +} + +type fakeSlackServer struct { + t *testing.T + mu sync.Mutex + srv *httptest.Server + + // script decides each call's reply. Replaced under the lock by the + // test as it moves the fake provider between health states. + script func(method string, call *e2eSlackCall) e2eSlackReply + calls []e2eSlackCall + nextTS int +} + +func newFakeSlackServer(t *testing.T) *fakeSlackServer { + t.Helper() + f := &fakeSlackServer{t: t, script: func(string, *e2eSlackCall) e2eSlackReply { return e2eSlackReply{} }} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeSlackServer) url() string { return f.srv.URL } + +// setScript replaces the fake provider's behavior. Every test moves the +// provider between health states through this one seam. +func (f *fakeSlackServer) setScript(script func(method string, call *e2eSlackCall) e2eSlackReply) { + f.mu.Lock() + defer f.mu.Unlock() + f.script = script +} + +// alwaysOK is the healthy provider. +func alwaysOK(string, *e2eSlackCall) e2eSlackReply { return e2eSlackReply{} } + +// alwaysStatus is a provider that answers every call with one HTTP status — +// 503 models a total outage, 429 a rate limit. +func alwaysStatus(status, retryAfter int) func(string, *e2eSlackCall) e2eSlackReply { + return func(string, *e2eSlackCall) e2eSlackReply { + return e2eSlackReply{HTTPStatus: status, RetryAfterSeconds: retryAfter} + } +} + +// alwaysError is a provider that rejects every call with one Slack error +// code (invalid_auth, channel_not_found, ...). +func alwaysError(code string) func(string, *e2eSlackCall) e2eSlackReply { + return func(string, *e2eSlackCall) e2eSlackReply { return e2eSlackReply{ErrorCode: code} } +} + +func (f *fakeSlackServer) handle(w http.ResponseWriter, r *http.Request) { + method := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + call := e2eSlackCall{Method: method, ClientMsgID: r.Header.Get("X-Alertint-Client-Message-Id")} + + if method != "auth.test" { + body, _ := io.ReadAll(r.Body) + var payload map[string]any + if err := json.Unmarshal(body, &payload); err == nil { + call.Channel, _ = payload["channel"].(string) + call.ThreadTS, _ = payload["thread_ts"].(string) + call.Broadcast, _ = payload["reply_broadcast"].(bool) + call.TS, _ = payload["ts"].(string) + call.Text, _ = payload["text"].(string) + } + } + + f.mu.Lock() + reply := f.script(method, &call) + f.nextTS++ + // chat.update answers with the timestamp of the message it edited, not + // a new one — exactly as Slack does. Getting this wrong would let every + // root edit silently rewrite the Situation's own durable root + // coordinates, which is precisely the corruption these tests exist to + // rule out. + ts := fmt.Sprintf("1700000%03d.000100", f.nextTS) + if method == "chat.update" && call.TS != "" { + ts = call.TS + } + call.Accepted = !reply.Drop && reply.ErrorCode == "" && reply.HTTPStatus == 0 + if call.Accepted && method == "chat.postMessage" { + call.TS = ts + } + f.calls = append(f.calls, call) + f.mu.Unlock() + + switch { + case reply.Drop: + // The provider accepted the message; the connection dies before the + // client can read the answer. This is the "uncertain success" the + // spec's at-least-once boundary is about. + hijacker, ok := w.(http.Hijacker) + if !ok { + f.t.Errorf("httptest ResponseWriter does not support Hijack; cannot model a lost response") + return + } + conn, _, err := hijacker.Hijack() + if err != nil { + f.t.Errorf("hijack: %v", err) + return + } + _ = conn.Close() + return + case reply.HTTPStatus != 0: + if reply.RetryAfterSeconds > 0 { + w.Header().Set("Retry-After", fmt.Sprint(reply.RetryAfterSeconds)) + } + w.WriteHeader(reply.HTTPStatus) + return + case reply.ErrorCode != "": + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"ok":false,"error":%q}`, reply.ErrorCode) + return + default: + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"ok":true,"channel":%q,"ts":%q}`, e2eChannel, ts) + } +} + +func (f *fakeSlackServer) snapshot() []e2eSlackCall { + f.mu.Lock() + defer f.mu.Unlock() + return append([]e2eSlackCall(nil), f.calls...) +} + +// accepted returns only the calls the provider actually accepted, i.e. the +// messages that really exist in the channel. +func (f *fakeSlackServer) accepted() []e2eSlackCall { + var out []e2eSlackCall + for _, c := range f.snapshot() { + if c.Accepted && c.Method != "auth.test" { + out = append(out, c) + } + } + return out +} + +// ---------------------------------------------------------------------- +// Fixture: real store, real controller, real worker, real deliverer. +// ---------------------------------------------------------------------- + +const ( + e2eChannel = "C-E2E" + e2eOwner = "e2e" +) + +type e2eClock struct { + mu sync.Mutex + now time.Time +} + +func (c *e2eClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *e2eClock) advance(d time.Duration) { + c.mu.Lock() + c.now = c.now.Add(d) + c.mu.Unlock() +} + +type e2eFixture struct { + t *testing.T + ctx context.Context //nolint:containedctx // test fixture only: always context.Background(), threaded through the helpers rather than repeated on each. + st *store.Store + clock *e2eClock + slack *fakeSlackServer + + worker *situation.NotificationWorker + l2 *e2eAssessmentClient +} + +func newE2EFixture(t *testing.T) *e2eFixture { + t.Helper() + ctx := context.Background() + st, err := store.Open(ctx, filepath.Join(t.TempDir(), "e2e.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + + clock := &e2eClock{now: time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC)} + fake := newFakeSlackServer(t) + client := notifyslack.NewClient(notifyslack.Config{ + BotToken: "xoxb-e2e-test-token", BaseURL: fake.url(), TimeoutSeconds: 2, + }) + deliverer := NewSituationDeliverer(st, client, e2eChannel, clock.Now) + + f := &e2eFixture{t: t, ctx: ctx, st: st, clock: clock, slack: fake, l2: &e2eAssessmentClient{}} + f.worker = situation.NewNotificationWorker(st, deliverer, + situation.NotificationWorkerConfig{Owner: e2eOwner + ":notify"}, clock.Now, + slog.New(slog.DiscardHandler)) + return f +} + +// e2eAssessmentClient answers every L2 dispatch with one accepted, schema- +// valid proposal. attention/claimReason steer the derived Operator contract +// exactly the way internal/situation's own replay fixture does. +type e2eAssessmentClient struct { + mu sync.Mutex + attention model.Attention + claimReason bool +} + +func (c *e2eAssessmentClient) steer(attention model.Attention, claimReason bool) { + c.mu.Lock() + c.attention, c.claimReason = attention, claimReason + c.mu.Unlock() +} + +func (c *e2eAssessmentClient) CompleteOnce(_ context.Context, _ string, prompt llm.Prompt, _ []string) (llm.OneShotCompletion, error) { + c.mu.Lock() + attention, claimReason := c.attention, c.claimReason + c.mu.Unlock() + + proposal := model.AssessmentProposal{ + SchemaVersion: model.AssessmentSchemaVersion, + Persistence: model.PersistenceSustained, + Impact: model.ImpactSuspected, + Novelty: model.NoveltyFamiliar, + Causality: model.CausalityCorrelated, + Attention: model.AttentionObserve, + } + if attention != "" { + proposal.Attention = attention + } + if claimReason { + if cand, ok := e2eFirstNonFloorCandidate(prompt); ok { + proposal.SufficientReason = &model.SufficientReason{ + Code: cand.Code, CandidateID: cand.ID, Summary: "Elapsed duration is an outlier for this group.", + } + } + } + raw, err := json.Marshal(proposal) + if err != nil { + return llm.OneShotCompletion{}, err + } + return llm.OneShotCompletion{ + Completion: llm.Completion{Raw: raw, Model: "e2e-model"}, + RequestStarted: llm.RequestStartStatusTrue, + }, nil +} + +// e2eFirstNonFloorCandidate reads the eligible Sufficient-reason candidates +// back out of the prompt the controller actually rendered — the only way a +// proposal can name a candidate that validation will accept. +func e2eFirstNonFloorCandidate(prompt llm.Prompt) (model.ReasonCandidate, bool) { + const marker = "Situation snapshot:\n" + idx := strings.Index(prompt.Prefix, marker) + if idx < 0 { + return model.ReasonCandidate{}, false + } + var dto struct { + EligibleReasons []model.ReasonCandidate `json:"eligible_reasons"` + } + if err := json.NewDecoder(strings.NewReader(prompt.Prefix[idx+len(marker):])).Decode(&dto); err != nil { + return model.ReasonCandidate{}, false + } + for _, c := range dto.EligibleReasons { + if !c.DeterministicFloor { + return c, true + } + } + return model.ReasonCandidate{}, false +} + +// seed creates one Situation for groupKey through the real Incident + +// situation-input round trip, then runs one controller cycle so it has a +// first authoritative Transition, an Episode summary, and its publication +// intents. Returns the Situation ID. +func (f *e2eFixture) seed(groupKey string) string { + f.t.Helper() + seedControllerRuntimeSituation(f.t, f.st, groupKey, f.clock.Now()) + // Resolve by INCIDENT, not by group key: a group may already carry + // several terminal Situations (seedPriorTerminalLineage), and + // seedControllerRuntimeSituation's own group-key lookup would then + // return an arbitrary one of them. + sitID := f.situationIDForIncident("inc-" + groupKey) + f.controllerCycle() + return sitID +} + +// controllerCycle drains every due Situation through the real controller +// worker at the current clock. +func (f *e2eFixture) controllerCycle() int { + f.t.Helper() + f.clock.advance(time.Minute) + cw := situation.NewControllerWorker(f.st, f.st, f.l2, situation.ControllerConfig{}, + situation.ControllerWorkerConfig{Owner: e2eOwner + ":controller", Now: f.clock.Now}, + f.clock.Now, audit.New(f.st.DB()), slog.New(slog.DiscardHandler)) + n, err := cw.Drain(f.ctx) + if err != nil { + f.t.Fatalf("controller drain: %v", err) + } + return n +} + +// deliverRound runs exactly one real notification-worker round. +func (f *e2eFixture) deliverRound() int { + f.t.Helper() + n, err := f.worker.RunOnce(f.ctx) + if err != nil { + f.t.Fatalf("notification worker round: %v", err) + } + return n +} + +// deliverUntilQuiet runs delivery rounds, advancing the clock past each +// retry backoff, until a round handles nothing. +// It requires TWO consecutive empty rounds with a clock advance between +// them: one empty round only proves nothing is claimable at this instant, +// which is also exactly what a pending retry backoff looks like. +func (f *e2eFixture) deliverUntilQuiet(maxRounds int) { + f.t.Helper() + empty := 0 + for i := 0; i < maxRounds; i++ { + if f.deliverRound() == 0 { + empty++ + if empty >= 2 { + return + } + } else { + empty = 0 + } + f.clock.advance(6 * time.Minute) // past the 5m retry ceiling + } + f.t.Fatalf("delivery did not reach quiescence within %d rounds: %s", maxRounds, f.intentSummary()) +} + +// ---------------------------------------------------------------------- +// Durable-state readers. +// ---------------------------------------------------------------------- + +type e2eIntent struct { + ID string + Class string + Status string + ClientMessageID string + AttemptCount int + ErrorClass string + RetryAt string + DeliveredAs string + MessageTS string + Sequence int +} + +func (f *e2eFixture) intents() []e2eIntent { + f.t.Helper() + rows, err := f.st.DB().QueryContext(f.ctx, ` + SELECT i.id, i.effect_class, i.status, i.client_message_id, i.attempt_count, + COALESCE(i.last_error_class,''), COALESCE(i.retry_at,''), + COALESCE(i.delivered_as,''), COALESCE(i.message_ts,''), + COALESCE(t.sequence, 0) + FROM notification_intents i + LEFT JOIN situation_transitions t ON t.id = i.transition_id + ORDER BY i.created_at ASC, COALESCE(t.sequence,0) ASC, i.effect_class ASC`) + if err != nil { + f.t.Fatalf("read intents: %v", err) + } + defer func() { _ = rows.Close() }() + var out []e2eIntent + for rows.Next() { + var i e2eIntent + if err := rows.Scan(&i.ID, &i.Class, &i.Status, &i.ClientMessageID, &i.AttemptCount, + &i.ErrorClass, &i.RetryAt, &i.DeliveredAs, &i.MessageTS, &i.Sequence); err != nil { + f.t.Fatalf("scan intent: %v", err) + } + out = append(out, i) + } + if err := rows.Err(); err != nil { + f.t.Fatalf("iterate intents: %v", err) + } + return out +} + +func (f *e2eFixture) intentSummary() string { + var b strings.Builder + for _, i := range f.intents() { + fmt.Fprintf(&b, "\n %s status=%s attempts=%d err=%s delivered_as=%s seq=%d", + i.Class, i.Status, i.AttemptCount, i.ErrorClass, i.DeliveredAs, i.Sequence) + } + return b.String() +} + +func (f *e2eFixture) intentsOfClass(class string) []e2eIntent { + var out []e2eIntent + for _, i := range f.intents() { + if i.Class == class { + out = append(out, i) + } + } + return out +} + +func (f *e2eFixture) scalarInt(query string, args ...any) int { + f.t.Helper() + var n int + if err := f.st.DB().QueryRowContext(f.ctx, query, args...).Scan(&n); err != nil { + f.t.Fatalf("query %q: %v", query, err) + } + return n +} + +func (f *e2eFixture) rootCoordinates(situationID string) (string, string) { + f.t.Helper() + var channel, ts *string + if err := f.st.DB().QueryRowContext(f.ctx, + `SELECT slack_channel, slack_root_ts FROM situations WHERE id = ?`, situationID).Scan(&channel, &ts); err != nil { + f.t.Fatalf("read root coordinates: %v", err) + } + if channel == nil || ts == nil { + return "", "" + } + return *channel, *ts +} + +// ---------------------------------------------------------------------- +// 1. Uncertain success: the response is lost after Slack accepted the post. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EUncertainPostReusesClientIdentityAndAcceptsADuplicate(t *testing.T) { + f := newE2EFixture(t) + + // The provider accepts the first post and then loses the connection, so + // the client can never learn the outcome. Every later call succeeds. + var dropped bool + f.slack.setScript(func(method string, _ *e2eSlackCall) e2eSlackReply { + if method == "chat.postMessage" && !dropped { + dropped = true + return e2eSlackReply{Drop: true} + } + return e2eSlackReply{} + }) + + sitID := f.seed("group=e2e-uncertain") + f.deliverUntilQuiet(12) + + roots := f.intentsOfClass("root_sync") + if len(roots) != 1 { + t.Fatalf("root_sync intents = %d, want exactly 1: an uncertain response must never create a second local intent%s", len(roots), f.intentSummary()) + } + if roots[0].Status != "delivered" { + t.Fatalf("root_sync status = %s, want delivered%s", roots[0].Status, f.intentSummary()) + } + + // The provider saw the root post TWICE, with the SAME client message id + // both times — the at-least-once external boundary ADR-0049 makes + // explicit, with a stable retry identity a deduplicating provider could + // use. + var rootPosts []e2eSlackCall + for _, c := range f.slack.snapshot() { + if c.Method == "chat.postMessage" && c.ThreadTS == "" { + rootPosts = append(rootPosts, c) + } + } + if len(rootPosts) != 2 { + t.Fatalf("root posts reaching the provider = %d, want 2 (one lost response, one retry)", len(rootPosts)) + } + if rootPosts[0].ClientMsgID == "" || rootPosts[0].ClientMsgID != rootPosts[1].ClientMsgID { + t.Fatalf("client message ids = %q and %q, want one identical non-empty id across the retry", + rootPosts[0].ClientMsgID, rootPosts[1].ClientMsgID) + } + if rootPosts[1].ClientMsgID != roots[0].ClientMessageID { + t.Fatalf("wire client message id %q != durable intent client_message_id %q", + rootPosts[1].ClientMsgID, roots[0].ClientMessageID) + } + + // The authoritative coordinates are the ones the SECOND (acknowledged) + // call returned; the lost first post never corrupted them. + channel, ts := f.rootCoordinates(sitID) + if channel != e2eChannel || ts != roots[0].MessageTS { + t.Fatalf("root coordinates = (%q,%q), want (%q,%q)", channel, ts, e2eChannel, roots[0].MessageTS) + } +} + +// ---------------------------------------------------------------------- +// 2. A failing root edit, a rate limit, and an hour of outage all retry +// indefinitely — a valid effect never exhausts. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2ERetriesIndefinitelyAndHonorsRetryAfter(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysOK) + sitID := f.seed("group=e2e-retry") + f.deliverUntilQuiet(12) + if _, ts := f.rootCoordinates(sitID); ts == "" { + t.Fatal("the first root never published; the rest of this test has no root to edit") + } + + // Make the next controller cycle material, so it produces a root EDIT + // (chat.update against the published coordinates) plus a journal entry. + f.l2.steer(model.AttentionInvestigate, false) + f.clock.advance(20 * time.Minute) + if n := f.controllerCycle(); n == 0 { + t.Fatal("no controller work was due; the scenario needs a second material cycle") + } + + // Slack is rate limiting with an explicit Retry-After. + f.slack.setScript(alwaysStatus(http.StatusTooManyRequests, 45)) + f.deliverRound() + + pending := f.pendingBesidesDelivered() + if len(pending) == 0 { + t.Fatalf("nothing is pending after a rate limit; a rate-limited effect must stay claimable%s", f.intentSummary()) + } + rateLimited := 0 + for _, i := range pending { + if i.Status != "pending" { + t.Fatalf("%s intent status = %s after a rate limit, want pending%s", i.Class, i.Status, f.intentSummary()) + } + if i.ErrorClass != "ratelimited" { + // A dependent journal entry that was never claimable this round + // (its root is still owed) has nothing to honor yet — that is + // the ordering rule working, not a missing backoff. + continue + } + rateLimited++ + if i.RetryAt == "" { + t.Fatalf("%s intent has no retry_at after a rate limit%s", i.Class, f.intentSummary()) + } + retryAt, err := time.Parse(time.RFC3339Nano, i.RetryAt) + if err != nil { + t.Fatalf("parse retry_at %q: %v", i.RetryAt, err) + } + if earliest := f.clock.Now().Add(45 * time.Second); retryAt.Before(earliest) { + t.Fatalf("%s retry_at = %s, want at or after the honored Retry-After %s", i.Class, retryAt, earliest) + } + } + if rateLimited == 0 { + t.Fatalf("no intent recorded the rate limit%s", f.intentSummary()) + } + + // A full hour of hard outage. Nothing may ever fail or block. + f.slack.setScript(alwaysStatus(http.StatusServiceUnavailable, 0)) + for elapsed := time.Duration(0); elapsed < time.Hour; elapsed += 5 * time.Minute { + f.deliverRound() + f.clock.advance(5 * time.Minute) + } + for _, i := range f.intents() { + if i.Status == "failed" || i.Status == "blocked_configuration" { + t.Fatalf("%s intent reached %s after an hour of outage; a valid effect must retry indefinitely%s", + i.Class, i.Status, f.intentSummary()) + } + } + attempts := 0 + for _, i := range f.intents() { + attempts += i.AttemptCount + } + if attempts == 0 { + t.Fatal("no delivery attempt was ever recorded during the outage") + } + + // Slack comes back: everything converges with no operator action. + f.slack.setScript(alwaysOK) + f.deliverUntilQuiet(20) + for _, i := range f.intents() { + if i.Status != "delivered" && i.Status != "superseded" && i.Status != "withheld_by_operator_slack_floor" { + t.Fatalf("%s intent status = %s after recovery, want delivered/superseded/withheld%s", i.Class, i.Status, f.intentSummary()) + } + } +} + +// pendingBesidesDelivered returns every intent not already delivered, +// superseded, or withheld — i.e. the durable work still owed to Slack. +func (f *e2eFixture) pendingBesidesDelivered() []e2eIntent { + var out []e2eIntent + for _, i := range f.intents() { + switch i.Status { + case "delivered", "superseded", "withheld_by_operator_slack_floor": + default: + out = append(out, i) + } + } + return out +} + +// ---------------------------------------------------------------------- +// 3. Definite configuration rejections block rather than exhaust. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EConfigurationRejectionsBlockAndNeverDiscard(t *testing.T) { + for _, tc := range []struct { + name string + code string + }{ + {"invalid_token", "invalid_auth"}, + {"channel_loss", "channel_not_found"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysError(tc.code)) + f.seed("group=e2e-" + tc.name) + + for i := 0; i < 4; i++ { + f.deliverRound() + f.clock.advance(6 * time.Minute) + } + + blocked := f.scalarInt(`SELECT COUNT(*) FROM notification_intents WHERE status = 'blocked_configuration'`) + if blocked == 0 { + t.Fatalf("no intent reached blocked_configuration after %q%s", tc.code, f.intentSummary()) + } + if failed := f.scalarInt(`SELECT COUNT(*) FROM notification_intents WHERE status = 'failed'`); failed != 0 { + t.Fatalf("%d intent(s) reached failed; a configuration rejection must block, never exhaust%s", failed, f.intentSummary()) + } + // The durable obligation survives: correcting configuration + // returns the blocked work to pending and it delivers. + f.slack.setScript(alwaysOK) + if _, err := f.worker.ReactivateConfiguration(f.ctx); err != nil { + t.Fatalf("reactivate configuration: %v", err) + } + f.deliverUntilQuiet(12) + if remaining := len(f.pendingBesidesDelivered()); remaining != 0 { + t.Fatalf("%d intent(s) still owed after corrected configuration%s", remaining, f.intentSummary()) + } + }) + } +} + +// ---------------------------------------------------------------------- +// 4. A six-minute outage opens one gap generation; recovery emits exactly +// one System notice ahead of a complete, ordered replay; a second +// outage during that replay opens a second generation. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EOutageOpensGapRecoversAndReplaysInOrder(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysStatus(http.StatusServiceUnavailable, 0)) + + // Two Situations accumulate distinct journal entries while Slack is + // unavailable, so the replay has real per-episode history to order. + a := f.seed("group=e2e-gap-a") + b := f.seed("group=e2e-gap-b") + f.deliverRound() + + // Hold the outage past the five-minute threshold. + for elapsed := time.Duration(0); elapsed < 6*time.Minute; elapsed += time.Minute { + f.clock.advance(time.Minute) + f.deliverRound() + } + f.l2.steer(model.AttentionInvestigate, false) + f.clock.advance(20 * time.Minute) + f.controllerCycle() + f.deliverRound() + + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 1 { + t.Fatalf("delivery gap generations = %d, want exactly 1 after one continuous outage%s", gaps, f.intentSummary()) + } + if open := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_state WHERE open_gap_generation IS NOT NULL`); open != 1 { + t.Fatal("the open gap generation is not recorded on the installation delivery state") + } + if notices := f.scalarInt(`SELECT COUNT(*) FROM notification_intents WHERE effect_class = 'installation_gap_recovery'`); notices != 0 { + t.Fatalf("%d recovery notice(s) exist while the gap is still open; the notice is created on recovery only", notices) + } + + // Slack comes back. The next probe recovers the generation and the + // bounded System notice becomes claimable ahead of the backlog. + f.slack.setScript(alwaysOK) + f.clock.advance(6 * time.Minute) + f.deliverUntilQuiet(40) + + notices := f.intentsOfClass("installation_gap_recovery") + if len(notices) != 1 { + t.Fatalf("installation_gap_recovery intents = %d, want exactly one bounded notice per generation%s", len(notices), f.intentSummary()) + } + if notices[0].Status != "delivered" { + t.Fatalf("the recovery notice status = %s, want delivered", notices[0].Status) + } + + calls := f.slack.accepted() + noticeIdx := -1 + for i, c := range calls { + if strings.Contains(c.Text, "AlertINT's Slack delivery was interrupted") { + noticeIdx = i + break + } + } + if noticeIdx < 0 { + t.Fatalf("no System recovery notice reached the channel; accepted calls = %d", len(calls)) + } + if noticeIdx != 0 { + t.Fatalf("the System recovery notice is accepted call #%d, want the first message of the replay", noticeIdx+1) + } + + // Every affected Situation's root published, and its journal entries + // replayed under that root in Transition-sequence order. + for _, sit := range []string{a, b} { + channel, ts := f.rootCoordinates(sit) + if channel == "" || ts == "" { + t.Fatalf("situation %s has no published root after replay", sit) + } + assertJournalRepliesInSequenceOrder(t, f, sit, ts) + } + + // A second outage during ongoing delivery opens a SECOND generation — + // generations are per continuous outage, never reused. + f.l2.steer(model.AttentionObserve, false) + f.slack.setScript(alwaysStatus(http.StatusServiceUnavailable, 0)) + f.clock.advance(20 * time.Minute) + f.controllerCycle() + for elapsed := time.Duration(0); elapsed < 7*time.Minute; elapsed += time.Minute { + f.clock.advance(time.Minute) + f.deliverRound() + } + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 2 { + t.Fatalf("delivery gap generations = %d after a second outage, want 2%s", gaps, f.intentSummary()) + } + + f.slack.setScript(alwaysOK) + f.clock.advance(6 * time.Minute) + f.deliverUntilQuiet(40) + if notices := f.scalarInt( + `SELECT COUNT(*) FROM notification_intents WHERE effect_class = 'installation_gap_recovery'`); notices != 2 { + t.Fatalf("recovery notices = %d after two generations, want exactly one per generation%s", notices, f.intentSummary()) + } +} + +// assertJournalRepliesInSequenceOrder proves every delivered journal entry +// for situationID reached the provider under that Situation's own root, in +// Transition-sequence order: a later entry can never pass an earlier one. +func assertJournalRepliesInSequenceOrder(t *testing.T, f *e2eFixture, situationID, rootTS string) { + t.Helper() + rows, err := f.st.DB().QueryContext(f.ctx, ` + SELECT t.sequence, i.message_ts + FROM notification_intents i + JOIN situation_transitions t ON t.id = i.transition_id + WHERE i.situation_id = ? AND i.status = 'delivered' + AND i.effect_class IN ('thread_append','broadcast_handoff') + ORDER BY t.sequence ASC`, situationID) + if err != nil { + t.Fatalf("read delivered journal intents: %v", err) + } + defer func() { _ = rows.Close() }() + type entry struct { + sequence int + ts string + } + var entries []entry + for rows.Next() { + var e entry + if err := rows.Scan(&e.sequence, &e.ts); err != nil { + t.Fatalf("scan journal intent: %v", err) + } + entries = append(entries, e) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate journal intents: %v", err) + } + if len(entries) == 0 { + return + } + + arrival := map[string]int{} + for i, c := range f.slack.accepted() { + if c.Method == "chat.postMessage" && c.ThreadTS == rootTS { + arrival[c.TS] = i + } + } + last := -1 + for _, e := range entries { + idx, ok := arrival[e.ts] + if !ok { + t.Fatalf("journal entry for transition #%d (ts %s) never arrived under root %s", e.sequence, e.ts, rootTS) + } + if idx < last { + t.Fatalf("journal entry for transition #%d arrived before an earlier entry; replay must preserve Transition-sequence order", e.sequence) + } + last = idx + } +} + +// ---------------------------------------------------------------------- +// 5. A publication that was only ever queued, followed by terminal state, +// posts ONE latest-informative terminal root plus its ordered journal — +// outside any recorded gap. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EQueuedPublicationTerminalBeforeFirstSlackCall(t *testing.T) { + f := newE2EFixture(t) + // The provider is unreachable only long enough to keep the first root + // queued; it never fails long enough to open a gap generation. + f.slack.setScript(alwaysStatus(http.StatusServiceUnavailable, 0)) + + sitID := f.seed("group=e2e-queued-terminal") + f.deliverRound() + if _, ts := f.rootCoordinates(sitID); ts != "" { + t.Fatal("the root published despite an unreachable provider; this scenario needs it queued") + } + + // Terminalize the Situation before any Slack call succeeded. + f.terminalize(sitID) + + f.slack.setScript(alwaysOK) + f.clock.advance(time.Minute) + f.deliverUntilQuiet(40) + + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 0 { + t.Fatalf("delivery gap generations = %d, want 0: a delay under five minutes is an ordinary delay, not a gap", gaps) + } + + channel, rootTS := f.rootCoordinates(sitID) + if channel != e2eChannel || rootTS == "" { + t.Fatalf("root coordinates = (%q,%q); a queued publication must still publish its latest informative root", channel, rootTS) + } + + // Exactly one root message exists in the channel — the terminal one — + // and it is the LATEST informative projection, not the original + // pre-terminal card replayed as though current. + var rootPosts, edits int + for _, c := range f.slack.accepted() { + switch { + case c.Method == "chat.postMessage" && c.ThreadTS == "": + rootPosts++ + case c.Method == "chat.update": + edits++ + } + } + if rootPosts != 1 { + t.Fatalf("root posts = %d, want exactly 1: a delayed publication posts one root, never two", rootPosts) + } + liveRoots := 0 + for _, i := range f.intentsOfClass("root_sync") { + if i.Status == "delivered" { + liveRoots++ + } + } + if liveRoots == 0 { + t.Fatalf("no root_sync intent delivered%s", f.intentSummary()) + } + assertJournalRepliesInSequenceOrder(t, f, sitID, rootTS) + + terminal := f.scalarInt( + `SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ? AND lifecycle IN ('recovered','closed_unknown')`, sitID) + if terminal == 0 { + t.Fatal("the Situation never reached a terminal Transition") + } +} + +// terminalize drives situationID to closed_unknown through the real +// controller: its symptoms stop reporting and its source-aware +// lifecycle-observation deadline then expires. Nothing is hand-written. +func (f *e2eFixture) terminalize(situationID string) { + f.t.Helper() + // The seeded Incident carries no firing delivery of its own, so the + // only thing standing between it and closed_unknown is the + // observation deadline. The long duration class's deadline is 7 days. + f.clock.advance(8 * 24 * time.Hour) + if _, err := f.st.DB().ExecContext(f.ctx, + `UPDATE situations SET next_assessment_at = ? WHERE id = ?`, + f.clock.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano), situationID); err != nil { + f.t.Fatalf("make situation due: %v", err) + } + if n := f.controllerCycle(); n == 0 { + f.t.Fatal("no controller work was due; the Situation cannot terminalize") + } + var lifecycle string + if err := f.st.DB().QueryRowContext(f.ctx, `SELECT lifecycle FROM situations WHERE id = ?`, situationID).Scan(&lifecycle); err != nil { + f.t.Fatalf("read lifecycle: %v", err) + } + if lifecycle != "closed_unknown" && lifecycle != "recovered" { + f.t.Fatalf("situation lifecycle = %q, want a terminal one", lifecycle) + } +} + +// ---------------------------------------------------------------------- +// 6. A stale handoff is never broadcast as current: its immutable history +// still delivers, demoted to a delayed non-current thread entry. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EStaleHandoffIsDemotedToADelayedThreadEntry(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysOK) + // Five completed Situations for this group first: duration_outlier — + // the only non-floor Sufficient-reason candidate this build can reach, + // and therefore the only path to an operator handoff — needs at least + // five comparable prior durations. + f.seedPriorTerminalLineage("group=e2e-stale-handoff", 5) + sitID := f.seed("group=e2e-stale-handoff") + f.deliverUntilQuiet(12) + + // Hand off to the operator: this creates the one broadcast_handoff + // effect the spec allows. + f.l2.steer(model.AttentionInvestigate, true) + f.clock.advance(3 * time.Hour) + f.controllerCycle() + + if handoffs := f.intentsOfClass("broadcast_handoff"); len(handoffs) == 0 { + t.Fatalf("no broadcast_handoff intent was created; the operator handoff never happened%s", f.intentSummary()) + } + + // Before it can be delivered, the required action stops being current. + f.l2.steer(model.AttentionObserve, false) + f.clock.advance(20 * time.Minute) + f.controllerCycle() + + f.deliverUntilQuiet(20) + + delivered := f.intentsOfClass("broadcast_handoff") + if len(delivered) == 0 { + t.Fatal("the broadcast handoff intent disappeared") + } + for _, i := range delivered { + if i.Status != "delivered" { + t.Fatalf("broadcast_handoff status = %s, want delivered: immutable history is never dropped%s", i.Status, f.intentSummary()) + } + if i.DeliveredAs != "delayed_thread" { + t.Fatalf("broadcast_handoff delivered_as = %q, want delayed_thread: a stale handoff must never broadcast as current", i.DeliveredAs) + } + } + _, rootTS := f.rootCoordinates(sitID) + if rootTS == "" { + t.Fatal("the Situation has no published root; its demoted history has nothing to hang under") + } + for _, c := range f.slack.accepted() { + if c.Broadcast { + t.Fatal("a reply reached the main channel as a broadcast after its requested action stopped being current") + } + if c.Method == "chat.postMessage" && c.ThreadTS != "" && c.ThreadTS != rootTS { + t.Fatalf("a journal reply was threaded under %q, not this Situation's own root %q", c.ThreadTS, rootTS) + } + } +} + +// seedPriorTerminalLineage records n completed Situations for groupKey so +// the live one's duration_outlier candidate — this build's only reachable +// non-floor Sufficient reason — becomes admissible. They are written +// directly because they are HISTORY, not the subject under test: what this +// file exercises is delivery, and internal/situation's own +// history_replay_test.go already drives the identical lineage end to end +// through real HTTP ingestion. +func (f *e2eFixture) seedPriorTerminalLineage(groupKey string, n int) { + f.t.Helper() + for i := 0; i < n; i++ { + // One at a time: migration 0014 allows exactly one nonterminal + // Situation per group, so each prior is created through the same + // production Incident + situation-input path every other Situation + // in this file uses, then closed before the next one starts. + f.closeSituation(f.seedRawSituation(groupKey, fmt.Sprintf("prior-%d", i))) + } +} + +// seedRawSituation creates one additional Situation for groupKey through +// the real Incident + situation-input round trip, distinguished by suffix. +func (f *e2eFixture) seedRawSituation(groupKey, suffix string) string { + f.t.Helper() + incID := "inc-" + groupKey + "-" + suffix + now := f.clock.Now() + if err := f.st.InsertIncident(f.ctx, store.Incident{ + ID: incID, GroupKey: groupKey, FirstAlertAt: now, LastAlertAt: now, ReadyAt: now.Add(time.Minute), + }); err != nil { + f.t.Fatalf("insert prior incident: %v", err) + } + // Close its collecting window immediately: incidents carry a partial + // UNIQUE index on group_key while collecting, so the next prior cannot + // open until this one is ready. + if err := f.st.MarkIncidentReady(f.ctx, incID); err != nil { + f.t.Fatalf("mark prior incident ready: %v", err) + } + inputID := "input-" + groupKey + "-" + suffix + if _, err := f.st.DB().ExecContext(f.ctx, ` + INSERT INTO situation_input_outbox (id, idempotency_key, incident_id, kind, group_key, occurred_at, status) + VALUES (?, ?, ?, 'incident_created', ?, ?, 'pending')`, + inputID, "idem:"+inputID, incID, groupKey, now.UTC().Format(time.RFC3339Nano)); err != nil { + f.t.Fatalf("insert prior situation input: %v", err) + } + claims, err := f.st.ClaimSituationInputs(f.ctx, "e2e-prior:"+suffix, now, time.Minute, 1) + if err != nil || len(claims) != 1 { + f.t.Fatalf("claim prior situation input: claims=%d err=%v", len(claims), err) + } + if err := f.st.ApplySituationInput(f.ctx, claims[0]); err != nil { + f.t.Fatalf("apply prior situation input: %v", err) + } + return f.situationIDForIncident(incID) +} + +// closeSituation closes one Situation as closed_unknown — the one terminal +// lifecycle reachable directly from active, so migration 0014's own +// transition trigger accepts it. This is fixture bookkeeping for HISTORY +// rows only: the Situation under test in every assertion below reaches +// every lifecycle it visits through the real controller, and +// internal/situation/history_replay_test.go drives the same lineage end to +// end through real HTTP ingestion. +func (f *e2eFixture) closeSituation(situationID string) { + f.t.Helper() + if _, err := f.st.DB().ExecContext(f.ctx, + `UPDATE situations SET lifecycle = 'closed_unknown', terminal_at = ?, + terminal_reason = 'observation_deadline' WHERE id = ?`, + f.clock.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), situationID); err != nil { + f.t.Fatalf("close situation: %v", err) + } +} + +func (f *e2eFixture) situationIDForIncident(incidentID string) string { + f.t.Helper() + var id string + if err := f.st.DB().QueryRowContext(f.ctx, + `SELECT situation_id FROM situation_incidents WHERE incident_id = ?`, incidentID).Scan(&id); err != nil { + f.t.Fatalf("find situation for incident %s: %v", incidentID, err) + } + return id +} + +// ---------------------------------------------------------------------- +// 7. A failing root EDIT never corrupts the published root coordinates and +// never blocks the Situation's later history from delivering once the +// edit finally lands. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EFailingRootUpdateKeepsCoordinatesAndRecovers(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysOK) + sitID := f.seed("group=e2e-root-update") + f.deliverUntilQuiet(12) + + _, publishedTS := f.rootCoordinates(sitID) + if publishedTS == "" { + t.Fatal("the first root never published") + } + + // Posts keep working; only the root EDIT fails, with a retryable + // Slack-side error. + var updateFailures int + f.slack.setScript(func(method string, _ *e2eSlackCall) e2eSlackReply { + if method == "chat.update" && updateFailures < 3 { + updateFailures++ + return e2eSlackReply{ErrorCode: "internal_error"} + } + return e2eSlackReply{} + }) + + f.l2.steer(model.AttentionInvestigate, false) + f.clock.advance(20 * time.Minute) + if n := f.controllerCycle(); n == 0 { + t.Fatal("no controller work was due; the scenario needs a second material cycle") + } + + f.deliverRound() + if _, ts := f.rootCoordinates(sitID); ts != publishedTS { + t.Fatalf("root coordinates moved to %q while the edit was failing, want the original %q", ts, publishedTS) + } + for _, i := range f.intentsOfClass("root_sync") { + if i.Status == "failed" || i.Status == "blocked_configuration" { + t.Fatalf("a retryable root edit reached %s%s", i.Status, f.intentSummary()) + } + } + + f.deliverUntilQuiet(20) + if updateFailures != 3 { + t.Fatalf("the fake provider rejected %d edits, want 3 — the scenario never exercised the failure", updateFailures) + } + if _, ts := f.rootCoordinates(sitID); ts != publishedTS { + t.Fatalf("root coordinates = %q after recovery, want the original %q: an edit never re-anchors a root", ts, publishedTS) + } + if remaining := len(f.pendingBesidesDelivered()); remaining != 0 { + t.Fatalf("%d effect(s) still owed after the edit recovered%s", remaining, f.intentSummary()) + } + assertJournalRepliesInSequenceOrder(t, f, sitID, publishedTS) +} diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index a148a49..42ed2b5 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -109,14 +109,16 @@ here too: storm collapse, known-issue short-circuits, and prompt selection. Every correlated delivery also feeds a **Situation** — a durable record that owns one or more Incidents under one exact group key across restarts, so a fresh firing of the same group finds its durable history waiting -rather than starting from nothing. Nothing closes a Situation in this -build: a group's Situation stays active indefinitely and keeps owning that -group's later Incidents, so a re-fire after a prior resolution feeds the -same Situation (or collapses into the judged Incident as an occurrence). -The episode-boundary machinery — a closed Situation refusing new work, a -fresh linked Situation opening in its place — is already enforced at the -storage layer, but only becomes observable once the controller ships -lifecycle termination. +rather than starting from nothing. In a released binary nothing closes a +Situation: a group's Situation stays active indefinitely and keeps owning +that group's later Incidents, so a re-fire after a prior resolution feeds +the same Situation (or collapses into the judged Incident as an +occurrence). The episode-boundary machinery — a closed Situation refusing +new work, a fresh linked Situation opening in its place — is enforced at +the storage layer there and becomes observable on the `state-controller` +branch, where the controller does terminate an episode (recovered, or +closed with uncertainty once the lifecycle-observation deadline expires) +and a later firing opens a fresh linked Situation. **Honest status:** the durable Situation foundation *and* a fenced Situation controller are real, integration-tested work landing on the @@ -131,17 +133,26 @@ cycle derives one authoritative Assessment (material facts, an operator-facing Attention level, and a bounded action contract) and is visible read-only through MCP (`alertint_list_situations`, `alertint_get_situation`) once it has run at least once for a Situation. +Also on that branch, every authoritative material change now commits one +**immutable Transition** and one version of a **current Episode summary** in +the same fenced transaction as the state it describes, together with every +notification intent that change warrants — and the Situation delivery worker +is the **only** Slack writer in runtime assembly. The Incident-keyed Slack +card, its resolve edit, and its recurrence replies described in +[Outbound notification](#8-outbound-notification) below are removed there; +Slack presents one Situation root plus an immutable ordered journal instead +(see [Slack](../notifications/slack.md#situation-owned-slack)). The two +`AlertINT system` installation messages — LLM dependency health and +Slack-delivery-gap recovery — are the only sanctioned exceptions. **Not yet wired, even on `state-controller`:** connector preparation for the controller's own evidence needs, durable Assessment/Triage artifacts beyond -the bounded recent-attempt history, immutable Transition/Episode summary -history, a Situation-owned Slack presence (the Slack card in Phase 1 below -is still keyed off the Incident, not the Situation), and the final v0.14 -cutover that would make this the only grouping/dispatch path. Everything in -Phase 1 below Correlation — memory, evidence, triage, verification, -notification — still runs exactly as described, keyed off the Incident, -unaffected by which Situation an Incident belongs to. There is no -`state_controller_mode`, shadow-output path, or legacy/new runtime switch: -one build runs one grouping/dispatch path at a time. +the bounded recent-attempt history, operator questions or judgments, and the +final v0.14 cutover that would make this the only grouping/dispatch path. +Everything in Phase 1 below Correlation — memory, evidence, triage, +verification — still runs keyed off the Incident, unaffected by which +Situation an Incident belongs to. There is no `state_controller_mode`, +shadow-output path, or legacy/new runtime switch: one build runs one +grouping/dispatch path, and one Slack writer, at a time. - **MCP tools:** `alertint_list_situations`, `alertint_get_situation` — see [MCP clients](../integrations/mcp-clients.md) @@ -163,17 +174,26 @@ one build runs one grouping/dispatch path at a time. exporter under [`telemetry.otlp`](../getting-started/configuration.md#telemetry) — no telemetry leaves the process by default. +- **Slack:** one root per published Situation plus an immutable ordered + journal, delivered from durable intents with indefinite retry, five-minute + Delivery gaps, and complete recovery replay — see + [Slack](../notifications/slack.md#situation-owned-slack) +- **stdout:** one `{"kind":"situation.transition",…}` line per committed + Transition, deduplicated by `transition_id`; it means the change is + durable, never that Slack has seen it - **Not yet:** connector preparation, Assessment/Triage artifacts beyond the - bounded recent-attempt history, Transition/Episode summary history, - Situation-owned Slack, OpenTelemetry metrics or logs export (traces only - today), the final v0.14 cutover + bounded recent-attempt history, operator questions or judgments, + OpenTelemetry metrics or logs export (traces only today), the final v0.14 + cutover ### 4. Memory Before spending an analysis, **AlertINT** checks whether it has seen this condition before. A re-fire of an already-analyzed group key inside the -collapse horizon attaches as an **occurrence** — the Slack card edits in -place, no second LLM call. A genuinely new incident whose key matches a past +collapse horizon attaches as an **occurrence** — no second LLM call; a +released binary edits the Incident card in place, and on the +`state-controller` branch the owning Situation's own journal carries the +recurrence milestone instead. A genuinely new incident whose key matches a past analysis gets the prior finding **recalled** into its prompt as a past hypothesis, never as evidence. See [incident memory](incident-memory.md). @@ -229,12 +249,22 @@ confidence. See [verification round](verification-round.md). ### 8. Outbound notification The final finding — the post-verification judgment, not the draft — is -emitted as one JSON line on stdout and, when configured, posted to a Slack -channel. When all alerts recover, **AlertINT** updates the original Slack -message in-place (🔴 → ✅) and posts a short resolution note in the thread. +emitted as one JSON line on stdout and, in a released binary, posted to a +Slack channel. When all alerts recover, that build updates the original +Slack message in-place (🔴 → ✅) and posts a short resolution note in the +thread. + +On the `state-controller` branch this Incident-keyed Slack path is removed +from runtime assembly entirely: the finding still reaches stdout, and Slack +is written only by the Situation delivery worker described in +[3a](#3a-situation-foundation-and-controller) — one root per Situation plus +an immutable ordered journal, from durable intents that retry indefinitely +(see [Slack](../notifications/slack.md#situation-owned-slack)). Nothing +in this build posts an Incident-shaped card, thread reply, or recurrence +reply any more. - **Method:** stdout (always available) and Slack Bot Token API - (`chat.postMessage` / `chat.update`) + (`chat.postMessage` / `chat.update`), written by exactly one path ## Phase 2 — Investigate diff --git a/docs/concepts/scope-and-limits.md b/docs/concepts/scope-and-limits.md index 3cc4489..171fe8f 100644 --- a/docs/concepts/scope-and-limits.md +++ b/docs/concepts/scope-and-limits.md @@ -56,17 +56,28 @@ is and isn't: at `awaiting_decision` until that controller requests, skips, or leaves it parked — nothing dispatches to the triage skill on its own anymore on that branch. + That branch also commits an **immutable Transition** and a versioned + **Episode summary** for every authoritative material change, in the same + fenced transaction as the state itself, and makes the Situation delivery + worker the only Slack writer in runtime assembly: Slack shows one + Situation root plus an immutable ordered journal, delivered from durable + intents that retry indefinitely, and no Incident-shaped card, resolve + edit, or recurrence reply is reachable there any more. Local delivery + intent is idempotent across crashes and restarts; **external delivery is + at-least-once**, so an uncertain Slack response followed by a retry can + rarely leave a duplicate message in the channel (AlertINT requests no + history-read scope and never reads the channel back to reconcile). - **Is not (yet), even on `state-controller`:** connector preparation for the controller's own evidence needs, durable Assessment/Triage artifacts - beyond the bounded recent-attempt history exposed over MCP, immutable - Transition/Episode summary history, or a Situation-owned Slack presence - (Slack, where enabled, still posts per-Incident, exactly as in Phase 1). + beyond the bounded recent-attempt history exposed over MCP, operator + questions, Situation judgments, or expected-behaviour envelopes. `alertint_get_situation` reads `assessment: null` and `operator_contract: null`, honestly, for any Situation the controller has not yet reconciled at least once — never a fabricated placeholder. - **No mode switch:** there is no `state_controller_mode`, shadow-output path, or legacy/new runtime toggle to configure — one build runs one - grouping/dispatch path at a time. + grouping/dispatch path, and one Slack writer, at a time. There is no dual + or shadow notification mode. ## Known weaknesses diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 1e1739e..ca5deb2 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -264,6 +264,16 @@ A released binary built from `main` does not read this section at all. | `retry.min_seconds` | int | `5` | Floor of the controller's transient-failure retry backoff (exponential from here, capped at `retry.max_seconds`). | | `retry.max_seconds` | int | `300` | Ceiling of the controller's transient-failure retry backoff. | | `retry.jitter_percent` | int | `20` | Jitter fraction (±) applied to the computed backoff, so concurrently-parked Situations don't all retry in lockstep. | +| `slack.repage_cooldown_seconds` | int | `900` | How long a *materially changed required action* must wait after a delivered main-channel interruption before it may create another one. It gates exactly that one case: a first publication, newly crossed criticality, newly urgent attention, and an operator hand-off all bypass it, because a cooldown must never swallow an escalation. | + +Slack delivery adds no other knobs. Retry timing (exponential 5 s → 5 min +with jitter, honouring a longer Slack `Retry-After`), batch size, and the +five-minute Delivery-gap threshold are protocol constants, and the delivery +worker reuses `lease_seconds`, `heartbeat_seconds`, and +`reconcile_poll_seconds` above. There is deliberately **no attempt ceiling +and no dead-letter setting**: a valid effect retries indefinitely, and a +definite configuration rejection blocks durably instead of being discarded +(see [Slack](../notifications/slack.md#delivery-durable-intent-indefinite-retry-at-least-once)). Cadence is persisted scheduling machinery, not card content — it only widens or narrows how soon the controller next reconciles a nonterminal @@ -359,11 +369,11 @@ starts when the aggregate LLM dependency state first becomes `degraded` or | Field | Type | Default | Description | |---|---|---|---| | `stdout` | bool | `true` | Deliver the finding to **stdout** as one JSON line. The full JSON is verbose detail: it is written **only at `--log-level=debug`** (consistently, in every format). At `info` the sink is still active — a send is confirmed on the `notified` line — but no JSON is written; the result shows as the one-line `finding` summary instead. Recommended to leave on. | -| `slack.enabled` | bool | `false` | Post a Block Kit message to a Slack channel via the bot-token API (message updated in-place on resolve) | -| `slack.bot_token_env` | string | — | Required when `slack.enabled: true`. Env var name holding the Slack bot token (`xoxb-…`, requires the `chat:write` scope) | +| `slack.enabled` | bool | `false` | Turn on Slack delivery. In a released binary this posts a Block Kit Incident card, updated in-place on resolve; on the `state-controller` branch it turns on the Situation delivery worker — the only Slack writer there — which posts one Situation root plus an immutable ordered journal thread. | +| `slack.bot_token_env` | string | — | Required when `slack.enabled: true`. Env var name holding the Slack bot token (`xoxb-…`, requires the `chat:write` scope; no history-read scope is ever requested) | | `slack.channel` | string | — | Required when `slack.enabled: true`. Channel name (e.g. `#alerts`) or ID (e.g. `C1234567890`) | -| `slack.min_severity` | string | `low` | Findings below this severity (`low` \| `medium` \| `high`) are not posted to Slack; stdout always emits. An incident suppressed at firing is also suppressed at resolution. The default posts everything. | -| `slack.recurrence_mode` | string | `change-gated` | How a recurring incident resurfaces in its thread: `change-gated` posts a thread reply only on a real-world change (severity rise, new symptom, faster cadence) or a milestone (×5/×10/×25/×50/×100, then every ×100) — replies stay in the thread, nothing extra is sent to the channel; `off` keeps recurrence to a silent card count-bump. See [Slack](../notifications/slack.md) for details. | +| `slack.min_severity` | string | `low` | The channel-noise floor (`low` \| `medium` \| `high`); stdout always emits regardless. In a released binary it compares against the finding's severity, and an incident suppressed at firing is also suppressed at resolution. On the `state-controller` branch it is the minimum **interruption priority** a *new* main-channel interruption must meet — never alert severity and never a model claim; `critical` always passes, a withheld interruption is durably recorded, and the floor never suppresses Situation state, MCP history, a root edit, or a journal reply. The default posts everything. | +| `slack.recurrence_mode` | string | `change-gated` | How a recurring incident resurfaces in its thread: `change-gated` posts a thread reply only on a real-world change (severity rise, new symptom, faster cadence) or a milestone (×5/×10/×25/×50/×100, then every ×100) — replies stay in the thread, nothing extra is sent to the channel; `off` keeps recurrence to a silent card count-bump. **No effect on the `state-controller` branch** (recurrence is carried by the owning Situation's own journal at the same milestone rungs); the key is still accepted so an existing config keeps loading. See [Slack](../notifications/slack.md) for details. | At startup the agent logs one `notifiers ready` line listing the active sinks (and the Slack channel) so you can see where findings will go. Every analysis @@ -382,6 +392,15 @@ finding: a recurrence attach (`"kind":"occurrence"`), an operator annotation (`"kind":"triage_exhausted"`, carrying the incident id, attempt count, and the last error). +On the `state-controller` branch the stdout stream additionally emits one +`{"kind":"situation.transition","version":1,…}` line for every committed +Situation Transition — identities, closed codes, hashes, counts, and instants +only, never prose. That line means the change is **durably committed**; it +never implies Slack delivery, and a quiet or floor-withheld Situation still +emits it. **Deduplicate by `transition_id`**: a consumer that restarts, or +reads a replayed stream, may legitimately see the same transition line more +than once. + See [Slack](../notifications/slack.md) for the full setup walkthrough. ## `mcp` diff --git a/docs/integrations/mcp-clients.md b/docs/integrations/mcp-clients.md index 43c34aa..bafd26b 100644 --- a/docs/integrations/mcp-clients.md +++ b/docs/integrations/mcp-clients.md @@ -145,8 +145,16 @@ restart Windsurf and check **Settings → MCP Servers**: | `alertint_recent_changes` | List recent deploys/releases/PRs matching a label selector (requires change enrichment enabled). | | `sentry_issues_list` | List live, distilled Sentry issues for a project (+ optional environment) by status (`unresolved`/`resolved`/`ignored`); requires the Sentry Error source enabled. | | `sentry_issues_trace` | Return full distilled stacktraces (`file:line`, function, `in_app`) for up to 10 Sentry issue ids; requires the Sentry Error source enabled. | -| `alertint_incident_annotate` | Attach a permanent, age-stamped operator note to an incident — context for the next investigator; never affects triage or memory recall. | -| `alertint_incident_capture_verdict` | Capture an operator-confirmed correction or confirmation as a replayable, graded record. A correction steers the next triage of its failure group (tested against live evidence, ruling-gated — never blended in) and demotes the corrected prior from strong recall; a confirmation retires steering. | +| `alertint_incident_annotate` | Attach a permanent, age-stamped operator note to an incident — context for the next investigator; never affects triage or memory recall. On the `state-controller` branch it is also journalled, attributed to the operator, into the owning Situation's Slack thread in the same transaction — as recorded context, never as a change to the assessment, the attention level, or publication authority, and never as proof that anyone touched the operated system. | +| `alertint_incident_capture_verdict` | Capture an operator-confirmed correction or confirmation as a replayable, graded record. A correction steers the next triage of its failure group (tested against live evidence, ruling-gated — never blended in) and demotes the corrected prior from strong recall; a confirmation retires steering. On the `state-controller` branch it is separately attributed in the owning Situation's journal and keeps exactly the authority it already had — no more. | + +Both feedback writes land whether or not a Situation currently owns the +incident. With **no current owner** — none was ever assigned, or the owner +had already closed — the write still persists and stays visible through the +incident's own history here and in the audit log; against an already-closed +Situation it appears in that Situation's `artifacts_recorded_after_closure`, +never journalled into the closed episode and never lost. No old Incident +Slack card is resurrected or rewritten either way. Read-only toward your systems, always; feedback writes (the last two tools above) land only in AlertINT's own incident state, additive and diff --git a/docs/notifications/slack.md b/docs/notifications/slack.md index fa1a78e..a41ecf7 100644 --- a/docs/notifications/slack.md +++ b/docs/notifications/slack.md @@ -1,6 +1,6 @@ --- title: "Slack" -description: "Send AlertINT findings to Slack channels." +description: "How AlertINT presents incidents and Situations in a Slack channel." section: "Notifications" order: 1 slug: "slack" @@ -8,31 +8,37 @@ slug: "slack" # Slack -**AlertINT** posts structured Block Kit messages to Slack after every -completed incident analysis. When alerts recover, the original message is -updated in-place and a thread reply is posted — one message per incident, -no channel noise. +**AlertINT** posts to one Slack channel over the bot-token Web API. What it +posts depends on which build you run: + +| Build | What Slack shows | Written by | +|---|---|---| +| Released binary (from `main`) | One **Incident card** per analyzed incident, edited in place on resolve, with a thread for detail | the Incident notification path | +| `state-controller` integration branch | One **Situation root** per Situation, edited in place, with an immutable ordered journal thread | the Situation delivery worker — the *only* Slack writer on that branch | + +On the integration branch the Incident-keyed card, its resolve edit, and its +recurrence replies are **gone**: runtime assembly no longer wires any +Incident-shaped Slack call at all. The two exceptions are installation-level +`AlertINT system` messages, described at the end of this page. + +Both models use the same app, token, channel, and setup below. There is no +mode switch: one build runs one Slack writer. Synthetic incidents fired by `alertint drill` are unmistakable in a shared -channel: every surface of their card — headline, thread details, and the -plain-text fallback — carries a 🧪 **DRILL** banner, so a drill never -reads as a real incident to a teammate scrolling past. +channel: every surface — headline, thread details, and the plain-text +fallback — carries a 🧪 **DRILL** banner, so a drill never reads as a real +incident to a teammate scrolling past. ## Setup — Slack app with bot token -Bot tokens let **AlertINT** track the message it posted. When an incident -fires, **AlertINT** posts a rich Block Kit message and records its position in -the channel. When all alerts recover, it updates that message in-place -(🔴 → ✅, a duration field appears) and posts a short resolution note in -the thread. - 1. **Create a Slack app.** Go to and click **Create New App → From scratch**. Name it **AlertINT** and select your workspace. 2. **Add the `chat:write` scope.** In the left sidebar, click **OAuth & Permissions**, scroll to **Bot Token Scopes**, and add - `chat:write`. That is the only permission **AlertINT** needs. + `chat:write`. That is the only permission **AlertINT** needs. It never + requests a history-read scope and never reads the channel back. 3. **Install to your workspace.** Scroll to the top of **OAuth & Permissions** and click **Install to Workspace → Allow**. @@ -62,16 +68,235 @@ the thread. channel: "#alerts" # channel name or ID where alerts should post ``` -What happens at runtime: +## Situation-owned Slack + +**Integration-branch behaviour, not yet the `main`-branch default.** This is +what a binary built from the `state-controller` branch posts. See +[Architecture: Situation foundation and +controller](../concepts/architecture.md#3a-situation-foundation-and-controller) +for where it sits in the pipeline. + +### One root, one thread + +A published Situation owns exactly **one** main-channel message — its +**root** — and an append-only thread of **journal entries** beneath it. + +- The **root** always states the *current* full picture: what is happening, + why the current attention level is warranted, what AlertINT checked or is + checking, who acts next, what happens next and by when, and the Situation's + immutable handle plus its MCP retrieval path. It is **edited in place** + every time that picture materially changes. Once terminal, it states what + happened, the final outcome, what AlertINT investigated or concluded, the + duration and peak attention, and any recorded operator involvement. +- Each **journal entry** is the immutable record of exactly one material + change, rendered from that change alone — never from the newest state. + Entries are non-broadcast replies: they stay in the thread. -- **Firing** — posts a brief main-channel message (name + root cause) and - immediately posts the full analysis — severity, confidence, correlation - findings, MCP hint — as a thread reply. -- **Resolved** — updates the original main-channel message in-place - (header changes 🔴 → ✅, duration appears) and posts full resolution - details — duration, alert count, resolved time — as a thread reply. +Journal entries are created for first publication, material investigation +changes and conclusions, operator-contract changes, recovery pending, +recovery refire, permitted recurrence milestones, recovery, closure with +uncertainty, and operator write-backs. Routine reconciliation, retry +accounting, and elapsed seconds ticking by create **nothing** — no entry, no +edit, no interruption. -## Message structure +### Orientation + +The root's first line is a compact orientation with exactly one phase +emphasised: + +```text +active, before material investigation + **Observed** → Investigating → Monitoring → Outcome + +active, once investigation starts or while work/action remains + Observed → **Investigating** → Monitoring → Outcome + +watching for sustained recovery + Observed → Investigating → **Monitoring** → Outcome + +recovered + Observed → Investigating → Monitoring → **Recovered** + +closed with uncertainty, with no monitoring phase in the episode + Observed → Investigating → **Closed uncertain** + +closed with uncertainty, after monitoring + Observed → Investigating → Monitoring → **Closed uncertain** +``` + +A Situation that closed uncertain without ever waiting out a stability grace +**omits** the Monitoring step rather than inventing it. A recovery that does +not hold returns the emphasis from Monitoring to Investigating. + +Orientation is a pure rendering of durable state — it is never stored as a +second lifecycle, never changed by delivery state, and moving between phases +never by itself creates a journal entry. + +The line immediately below renders the operator contract in plain prose, for +example `AlertINT is running Acute Triage · update by 10:00:15`. Every +instant uses Slack's viewer-local date markup with a UTC fallback, so the +time reads correctly in every reader's own timezone. A promised update that +has already passed renders as overdue, never as a current promise. + +### What earns a main-channel interruption + +A new main-channel interruption (a "poke") is permitted only for a first +warranted publication, newly crossed deterministic criticality, newly valid +urgent attention, a hand-off from "AlertINT is working on it" to "an operator +must act", or a materially changed required action once the repage cooldown +has elapsed. **Root edits and journal replies are never interruptions**, and +a missing acknowledgement never repeats one. A hand-off edits the root first, +then posts one broadcast reply. + +`notify.slack.min_severity` is the operator's floor on that interruption. In +the Situation path it compares against the derived **interruption priority** +(`critical` / `high` / `medium` / `low`) — not against alert severity and not +against anything the model said. `critical` always passes. The floor applies +only to a *new* main-channel poke: it never suppresses Situation state, the +history in MCP, a root edit, or a journal reply. A poke below the floor is +durably recorded as withheld, and later independently warranted +higher-priority state can still publish. + +### Delivery: durable intent, indefinite retry, at-least-once + +Every effect Slack must show is committed as a **durable notification intent** +in the same fenced database transaction as the change that warranted it. The +delivery worker is the only thing that talks to Slack, and it never holds a +database transaction open across a Slack call. + +- **Local delivery intent is idempotent.** Every post and reply carries a + deterministic client message ID derived from the intent's own identity. A + timeout, an uncertain success, a crash, or a restart reuses the identical + ID and payload. A restart never creates a second intent, a second root, or + a second interruption. +- **External delivery is at-least-once.** If Slack accepts a message but the + response is lost, AlertINT cannot tell that apart from a message that never + arrived, so it retries with the same identity. That can rarely produce a + **duplicate message in the channel**. AlertINT requests no Slack + history-read scope and performs no read-back reconciliation, so this + boundary is real and deliberate: a duplicate is preferred to a silently + missing notification. The durable coordinates and the ordered retry state + are never corrupted by it. +- **Valid effects retry indefinitely.** Retryable failures (transport + errors, 5xx, rate limits) back off exponentially from 5 seconds to 5 + minutes with jitter, honour a longer Slack `Retry-After`, and **never + exhaust**. There is no attempt ceiling and no dead-letter. +- **Definite configuration rejections block, they do not discard.** A + missing or invalid token, a lost channel, a revoked scope, or a permission + rejection moves the effect to `blocked_configuration`, where it waits + durably. Restarting with corrected configuration returns it to pending and + it delivers. Nothing is dropped to silence Slack. +- **Ordering is preserved.** A Situation's root must be durably delivered + before any reply is claimable; journal replies deliver in change order; a + hand-off's root edit delivers before its broadcast reply; and a stale root + projection is superseded before the call is made rather than posted and + then corrected. + +### Slack outages and recovery replay + +A first failing Slack call is an **ordinary delay** — one WARN in the console +action trail, then paced retry WARNs, and nothing in the channel. + +If Slack keeps failing continuously for **five minutes**, AlertINT opens one +durable **Delivery gap** generation. The gap makes the outage visible; it does +not change the delivery obligation. Subsequent failures join the same +generation. + +When Slack answers again, the generation recovers and AlertINT posts, in this +order: + +1. exactly **one bounded `AlertINT system` recovery notice** for that + generation, stating the interval the gap covered, how many Situations were + affected, how many effects were delayed, and that the backlog is now + replaying; then +2. for every affected Situation, its **latest informative root** — posted if + no root exists yet, edited if one does; then +3. **every** material journal entry, in change order. + +Replay is complete, not summarised: no episode and no journal entry is +dropped because the gap was long. Old calls to action are shown as delayed, +non-current thread history — a hand-off whose requested action is no longer +current is delivered as an ordinary non-broadcast entry marked delayed rather +than broadcast to the channel as though it were live. Rate limits and +ordering are still honoured, so a large backlog takes time to finish. + +A second outage during that replay opens its own generation with its own +single recovery notice; generations are never reused. + +If a Situation reached its terminal state while its first publication was +still queued, AlertINT posts **one** root — the latest informative terminal +one — and then the ordered journal. It never replays the original card as +though it were current, and never posts a generic "all clear". + +### Quiet Situations leave no Slack trace + +A Situation that never warrants publication posts nothing at all: no root, no +thread, no interruption. Its state, history, and delivery decisions remain +complete and readable over MCP and in the audit log. The same is true of a +Situation whose only poke was withheld by your `min_severity` floor. Silence +in Slack is never a gap in the record. + +### Operator notes and captured verdicts + +An annotation or a captured verdict recorded over MCP (see +[MCP clients](../integrations/mcp-clients.md)) is journalled into its +Situation's own thread, attributed to the operator, in the same transaction +that records it. + +- An **annotation** is recorded context. It does not change the assessment, + the attention level, the reason, or publication authority, and it is never + rendered as proof that anyone touched the operated system. +- A **captured verdict** is recorded with its own attribution and keeps + exactly the authority it already had — it can steer later triage through + the normal input path, and nothing more. +- If the incident has **no current owning Situation** — none was ever + assigned, or the owner had already closed — the write still lands and stays + visible through the incident's own MCP and audit history. It is recorded + against a closed Situation as an artifact that arrived after closure + (`artifacts_recorded_after_closure` in `alertint_get_situation`), never + journalled into a closed episode and never lost. No old Incident card is + resurrected or rewritten. + +### What Slack agreement is checked against + +Slack, SQLite, MCP, the audit log, logs, and stdout all identify the same +things by the same identities. `alertint_get_situation` exposes the whole +Slack presence of one Situation — whether a root is published, where it +lives, and every delivery obligation's class, status, priority, retry or +supersession reason, and delivered coordinates — and +`alertint_get_delivery_state` exposes the installation-level view: the +continuous-failure window, the configuration generation and how many effects +are blocked on it, the current Delivery gap and its backlog, and how many +outcomes Slack never confirmed either way. + +### stdout is independent of Slack + +Every material change also emits one machine-readable line on stdout: + +```json +{"kind":"situation.transition","version":1,"transition_id":"…","situation_id":"…","sequence":4,"reason":"recovery_observed","journal_kind":"recovery_pending","lifecycle":"recovery_pending","attention":"observe","next_actor":"alertint","drill":false} +``` + +This line means the change is durably committed. It does **not** mean +anything reached Slack — a quiet Situation, a floor-withheld poke, and a +Situation whose delivery is queued behind an outage all still emit state. +**Deduplicate by `transition_id`:** a consumer that restarts, or reads a +stream that was replayed, may see the same transition line more than once, +and the transition ID is the stable identity to key on. + +## Incident cards + +**Released-binary behaviour (builds from `main`).** On the integration branch +this whole surface is removed; everything below is what a released binary +does today. + +When an incident fires, **AlertINT** posts a brief main-channel message (name ++ root cause) and immediately posts the full analysis — severity, confidence, +correlation findings, MCP hint — as a thread reply. When all alerts recover, +it updates that message in place (🔴 → ✅, a duration field appears) and +posts a short resolution note in the thread. + +### Message structure Every notification uses Slack Block Kit. The same blocks appear for firing and resolved — only the header and fields change on resolution. @@ -90,10 +315,7 @@ and resolved — only the header and fields change on resolution. | Thread — agent handoff | The same handoff block, so the call to action reads identically on every firing surface. | | Thread — resolved | Posted when all alerts recover: duration, alert count, and resolved timestamp in a fields grid. | -## Example — firing - -Two messages are posted: a brief main-channel message and an immediate -thread reply with the full analysis. +### Example — firing Main channel: @@ -130,10 +352,7 @@ Operator notes 🤖 Investigate in your AI agent: investigate incident a1b2c3d4-5e6f-7a8b-9c0d-1e2f3a4b5c6d using alertint ``` -## Example — resolved - -The original main-channel message is updated in-place and a resolution -note is posted in the thread. +### Example — resolved Main channel (updated in-place): @@ -146,53 +365,51 @@ error rates and response latency across the cluster. Incident a1b2c3d4 · resolved after 15m · 14:52 UTC ``` -Thread reply: - -```text -✅ All clear — all alerts have recovered. - -Duration: 15m Alerts: 3 recovered Resolved: 14:52 UTC - -Incident a1b2c3d4 · duration 15m -``` - The MCP hint in the message footer is a pre-filled tool call. Paste it directly into Claude Code, Cursor, or Windsurf to open the full evidence pack for that incident — see [MCP clients](../integrations/mcp-clients.md). -## Recurrence resurfacing +### Recurrence resurfacing When an already-analyzed incident re-fires inside the collapse window, it doesn't get a new card — it attaches as another occurrence on the same incident, and the card that's already in the channel is what carries the -update. Recurrence never adds channel messages: everything below happens on -the existing card or inside its thread. What lands depends on what changed: - -- **A plain re-fire** — same symptom, same severity, steady cadence — just - bumps the occurrence count on the existing card in place - (`🔁 recurred ×N · last HH:MM`). No new message anywhere. -- **A real-world change** — severity escalated, a new symptom (alertname) - joined, or the cadence sped up markedly — posts a thread reply naming - exactly why (`why: severity` / `why: new_alertname` / `why: cadence`). - The escalation trigger is durably recorded on the occurrence, but the - automatic re-analysis it is meant to drive is not wired to the delivery - pipeline yet — acting on recorded triggers is an explicit obligation of - the upcoming Situation controller — so today the card's finding is not - re-edited on escalation; the thread reply is the escalation's visible - trace. -- **A steady flapper** that never trips one of those changes still gets a - thread reply at milestone counts — ×5, ×10, ×25, ×50, ×100, then every - ×100 — so a long-running recurring incident keeps a visible trail in its - thread even without a qualifying change. - -Every recurrence reply states the reason, e.g.: +update. Recurrence never adds channel messages: -```text -Incident a1b2c3d4 · recurred ×9 · last 14:52 UTC · why: cadence +- **A plain re-fire** just bumps the occurrence count on the existing card in + place (`🔁 recurred ×N · last HH:MM`). +- **A real-world change** — severity escalated, a new symptom joined, or the + cadence sped up markedly — posts a thread reply naming exactly why + (`why: severity` / `why: new_alertname` / `why: cadence`). +- **A steady flapper** still gets a thread reply at milestone counts — ×5, + ×10, ×25, ×50, ×100, then every ×100. + +Control this with `notify.slack.recurrence_mode`: + +```yaml +notify: + slack: + recurrence_mode: change-gated # change-gated (default) | off ``` +- `change-gated` (default) — post a thread reply on a real-world change or a + milestone, as described above. +- `off` — recurrence never posts replies; the card's occurrence count still + updates in place, silently. + +On the integration branch this setting has **no effect**: recurrence is +carried by the owning Situation's own journal at the same milestone rungs, so +a quiet Situation leaves no recurrence trace in Slack at all. The key is still +accepted so an existing `config.yaml` keeps loading. + ## System messages +Two installation-level `AlertINT system` messages are not tied to any +incident or Situation. They are the only sanctioned non-Situation Slack +writes on the integration branch. + +### LLM dependency health + The configured LLM is an installation dependency, not a property of any Incident. When it has been continuously unhealthy for `health.broadcast_after_seconds` (default 5 minutes), AlertINT posts one @@ -227,29 +444,13 @@ never reused by the next episode, which earns its own root on its own clock. Each Slack request is bounded by its own timeout: a request Slack accepts but never answers counts as an unknown outcome (a post) or is retried on the next minute (an edit), and never delays the idle probe that would report the -LLM back. During an outage, new Incident triage retries with -backoff and correlation may be delayed; this copy never claims Alert intake -itself is unaffected. See [Integration health](../getting-started/configuration.md#integration-health) +LLM back. During an outage, new Incident triage retries with backoff and +correlation may be delayed; this copy never claims Alert intake itself is +unaffected. See [Integration health](../getting-started/configuration.md#integration-health) for the `/health` shape behind these messages. -Two backstop triggers — a hard occurrence cap and a periodic re-analysis -ceiling — are likewise recorded without representing a genuine escalation, and -never post a reply; the fresh re-analysis they are meant to force awaits the -same Situation controller. - -Control this with `notify.slack.recurrence_mode`: - -```yaml -notify: - slack: - recurrence_mode: change-gated # change-gated (default) | off -``` - -- `change-gated` (default) — post a thread reply on a real-world change or a - milestone, as described above. -- `off` — recurrence never posts replies; the card's occurrence count still - updates in place, silently. +### Slack delivery gap recovery -Drill incidents (`alertint drill`) keep their 🧪 DRILL banner on every -recurrence surface — card edit, thread reply, and fallback text — same as -every other rendered surface. +**Integration branch only.** One bounded notice per Delivery-gap generation, +posted ahead of the backlog replay described above. It is not a Situation and +not an LLM message, and it is never edited or repeated. diff --git a/internal/situation/history_replay_test.go b/internal/situation/history_replay_test.go new file mode 100644 index 0000000..269444d --- /dev/null +++ b/internal/situation/history_replay_test.go @@ -0,0 +1,1486 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +// package situation_test (external test package) — see +// controller_replay_test.go's own header for the import-cycle constraint +// that forces every real-Store replay fixture in this package to live +// outside `package situation`. +// +// Plan 3 Task 10, Step 1: real-Store crash-boundary replay proof for +// Situation HISTORY. controller_replay_test.go already proves Plan 2's +// controller/Triage convergence across nine crash boundaries; this file +// reuses that file's fixture (replayFixture), its fault-injecting +// faultyControllerStore decorator, its crash points, and its +// simulateCrash harness verbatim — never a second harness — and extends +// them along the one axis Plan 3 added: the immutable Transition ledger, +// the versioned Episode-summary projection, the durable notification +// intents, the persisted Slack root coordinates, and the Delivery-gap +// generations. +// +// Every scenario below runs TWICE against two independent on-disk +// databases, through the same script: +// +// - the reference run restarts (close/reopen/advance) at each scripted +// boundary but never crashes; +// - the replay run crashes at that same boundary — inside the fenced +// controller commit, immediately after it, or between a delivery's +// Slack call and its durable acknowledgement — then restarts and +// replays. +// +// Both runs therefore see the IDENTICAL logical-clock schedule, so the +// only difference between them is the crash itself. (A reference run that +// simply never closed the database would drift on elapsed time, and +// elapsed time feeds DurationClass, which feeds MaterialFactHash — the +// comparison would then be measuring clock drift, not crash tolerance. +// See longClassMargin's own doc comment in controller_replay_test.go for +// the same hazard in Plan 2's fixture.) +// +// The two runs are then compared on canonicalHistory: the normalized, +// ID-independent projection of every Situation's Transition ledger, +// Episode summary, notification intents, root coordinates, and gap +// generations. Raw UUID/digest identities are normalized away (they are +// derived from the Situation ID, which differs between two independent +// databases by construction); everything an operator can actually observe +// — sequence, reason, journal kind, lifecycle, Attention, actor, poke +// authority, Interruption priority, intent status, summary version, +// whether a root is published — is compared verbatim. + +package situation_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/alertint/alertint-agent/internal/audit" + "github.com/alertint/alertint-agent/internal/llm" + "github.com/alertint/alertint-agent/internal/situation" + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" + "github.com/alertint/alertint-agent/internal/store" +) + +// ---------------------------------------------------------------------- +// Steerable L2 (Situation Assessment) client. +// +// controller_replay_test.go's newAcceptingL2Client always answers with the +// same observe-grade proposal, which is all Plan 2's convergence proofs +// needed. Plan 3's history catalog needs Attention to move and a +// Sufficient reason to be claimed, so this client reads the eligible +// reason candidates back out of the prompt the controller actually built +// (BuildAssessmentPrompt renders them into its snapshot body) and answers +// against them — exactly as a real model must, and with the same +// validation consequences: a candidate ID that is not in this snapshot's +// eligible_reasons is rejected as reason_id_unknown. +// ---------------------------------------------------------------------- + +// l2Script is what the fixture wants the next L2 answer to claim. The zero +// value is the plain observe-grade proposal newAcceptingL2Client returns. +type l2Script struct { + // Attention the proposal claims. Empty means observe. + // + // Note: a snapshot carrying a deterministic floor candidate + // (critical_anchor) has its Attention raised to urgent by + // validateProposalContent regardless of what is claimed here, and a + // claim of urgent WITHOUT such a floor is rejected outright + // (urgent_without_floor). This field therefore only ever moves + // Attention between observe and investigate. + Attention situationmodel.Attention + // ClaimNonFloorReason selects the first non-floor eligible candidate + // the prompt offers, when one exists, as the proposal's Sufficient + // reason. With Attention=investigate that is what makes the controller + // derive an operator handoff (assessment.go's operatorActionRequired: + // a validated, non-floor Sufficient reason accepted while Attention is + // investigate). + ClaimNonFloorReason bool +} + +type steerableL2Client struct { + t *testing.T + script func() l2Script + calls int +} + +func newSteerableL2Client(t *testing.T, script func() l2Script) *steerableL2Client { + t.Helper() + return &steerableL2Client{t: t, script: script} +} + +// promptSnapshotDTO is the narrow projection this client reads back out of +// the rendered prompt. It deliberately decodes only the two fields it +// needs — the prompt body carries the whole bounded snapshot, but a test +// client that decoded all of it would couple to every future snapshot +// field. +type promptSnapshotDTO struct { + EligibleReasons []situationmodel.ReasonCandidate `json:"eligible_reasons"` +} + +const promptSnapshotMarker = "Situation snapshot:\n" + +// eligibleReasonsFromPrompt decodes the eligible reason candidates out of +// the prompt the controller built. json.Decoder stops at the end of the +// first complete JSON value, so the schema instructions that follow the +// snapshot body in the same prefix are simply never read. +func eligibleReasonsFromPrompt(t *testing.T, prompt llm.Prompt) []situationmodel.ReasonCandidate { + t.Helper() + idx := strings.Index(prompt.Prefix, promptSnapshotMarker) + if idx < 0 { + t.Fatalf("assessment prompt does not carry the %q marker; the prompt shape changed", promptSnapshotMarker) + } + var dto promptSnapshotDTO + dec := json.NewDecoder(strings.NewReader(prompt.Prefix[idx+len(promptSnapshotMarker):])) + if err := dec.Decode(&dto); err != nil { + t.Fatalf("decode assessment prompt snapshot body: %v", err) + } + return dto.EligibleReasons +} + +func (c *steerableL2Client) CompleteOnce(_ context.Context, _ string, prompt llm.Prompt, _ []string) (llm.OneShotCompletion, error) { + c.calls++ + script := c.script() + + proposal := situationmodel.AssessmentProposal{ + SchemaVersion: situationmodel.AssessmentSchemaVersion, + Persistence: situationmodel.PersistenceSustained, + Impact: situationmodel.ImpactSuspected, + Novelty: situationmodel.NoveltyFamiliar, + Causality: situationmodel.CausalityCorrelated, + Attention: situationmodel.AttentionObserve, + } + if script.Attention != "" { + proposal.Attention = script.Attention + } + if script.ClaimNonFloorReason { + for _, cand := range eligibleReasonsFromPrompt(c.t, prompt) { + if cand.DeterministicFloor { + continue + } + proposal.SufficientReason = &situationmodel.SufficientReason{ + Code: cand.Code, + CandidateID: cand.ID, + Summary: "Elapsed duration is a statistical outlier against this group's own history.", + // Deliberately empty: every ref must exist in this exact + // snapshot's facts, and the candidate's own refs are the + // only set guaranteed to. Claiming none is always valid + // and keeps this client independent of fact identity. + EvidenceRefs: nil, + } + break + } + } + + raw, err := json.Marshal(proposal) + if err != nil { + c.t.Fatalf("marshal steered proposal: %v", err) + } + return llm.OneShotCompletion{ + Completion: llm.Completion{Raw: raw, Model: "history-replay-model", Latency: 5 * time.Millisecond}, + RequestStarted: llm.RequestStartStatusTrue, + }, nil +} + +// ---------------------------------------------------------------------- +// Notification delivery: a deterministic in-test deliverer plus the one +// extra crash boundary Plan 3 adds beyond Plan 2's controller boundaries — +// the process dying between a successful Slack call and the durable +// MarkNotificationDelivered that records its coordinates. +// +// This file's deliverer is intentionally trivial (it fabricates a +// coordinate, it renders nothing). Step 2's fake-Slack end-to-end test +// drives the REAL cmd/alertint SituationDeliverer against a real +// slack.Client and an httptest Slack server; this fixture's job is the +// STORE side of the boundary — what survives a crash and what replay +// converges to — which is exactly what a rendering-free deliverer isolates. +// ---------------------------------------------------------------------- + +type replayDeliverer struct { + // crashAfterCall makes Deliver panic AFTER it has decided its result + // but BEFORE returning it to the worker — modeling "Slack accepted the + // call, the process died before the acknowledgement was durable." + crashAfterCall bool + posted []string + seq int +} + +func (d *replayDeliverer) Probe(context.Context) error { return nil } + +func (d *replayDeliverer) Deliver(_ context.Context, intent situationmodel.NotificationIntent) (situation.NotificationDelivery, error) { + d.seq++ + d.posted = append(d.posted, intent.ClientMessageID) + delivery := situation.NotificationDelivery{ + Channel: "C-REPLAY", + MessageTS: fmt.Sprintf("17000000%02d.000100", d.seq), + DeliveredAs: deliveredAsFor(intent.EffectClass), + } + if d.crashAfterCall { + panic(replayCrash{boundary: crashBoundaryDeliveryAcknowledgement}) + } + return delivery, nil +} + +const crashBoundaryDeliveryAcknowledgement = "crash_after_slack_call_before_durable_acknowledgement" + +func deliveredAsFor(class situationmodel.EffectClass) string { + switch class { + case situationmodel.EffectRootSync: + return "root" + case situationmodel.EffectBroadcastHandoff: + return "broadcast" + case situationmodel.EffectInstallationGapRecovery: + return "system" + case situationmodel.EffectThreadAppend: + return "thread" + default: + return "thread" + } +} + +// ---------------------------------------------------------------------- +// historyFixture: replayFixture plus the Plan 3 driving surface. +// ---------------------------------------------------------------------- + +type historyFixture struct { + *replayFixture + + script l2Script + deliverer *replayDeliverer + // crashAt names the boundary this run crashes at, or "" for the + // reference (restart-only) run. + crashAt crashPoint + // crashDelivery makes the next delivery round crash between the Slack + // call and its durable acknowledgement. + crashDelivery bool +} + +func newHistoryFixture(t *testing.T, owner string, crashAt crashPoint, crashDelivery bool) *historyFixture { + t.Helper() + return &historyFixture{ + replayFixture: newReplayFixture(t, owner), + deliverer: &replayDeliverer{}, + crashAt: crashAt, + crashDelivery: crashDelivery, + } +} + +func (f *historyFixture) l2() *steerableL2Client { + return newSteerableL2Client(f.t, func() l2Script { return f.script }) +} + +// postAlert POSTs one Alertmanager v4 alert over real HTTP through the real +// Receiver, with an explicit status and severity label — replayFixture's own +// postGroup always posts a firing alert with no severity, which is all Plan +// 2's fixture needed. Severity matters here because a critical-severity +// delivery makes critical_anchor eligible, which is a DETERMINISTIC FLOOR: +// validateProposalContent then raises Attention to urgent unconditionally, +// and operatorActionRequired never fires for a floor candidate — so the +// handoff scenario must post a non-critical alert. +// +//nolint:unparam // alertname is a general fixture parameter; every current scenario happens to use "HighLatency". +func (f *historyFixture) postAlert(group, alertname, fingerprint, status, severity string) { + f.t.Helper() + labels := map[string]string{"alertname": alertname, "group": group} + if severity != "" { + labels["severity"] = severity + } + alert := map[string]any{ + "status": status, + "labels": labels, + "annotations": map[string]string{}, + "startsAt": f.clock.Now().Format(time.RFC3339Nano), + "fingerprint": fingerprint, + } + if status == "resolved" { + alert["endsAt"] = f.clock.Now().Format(time.RFC3339Nano) + } + f.postJSON(map[string]any{ + "version": "4", + "status": status, + "groupLabels": map[string]string{"group": group}, + "alerts": []map[string]any{alert}, + }) +} + +func (f *historyFixture) postJSON(payload map[string]any) { + f.t.Helper() + body, err := json.Marshal(payload) + if err != nil { + f.t.Fatalf("marshal alertmanager payload: %v", err) + } + req, err := http.NewRequestWithContext(f.ctx, http.MethodPost, f.srv.URL+"/webhook/alertmanager", bytes.NewReader(body)) + if err != nil { + f.t.Fatalf("new request: %v", err) + } + req.Header.Set("Authorization", "Bearer "+replayToken) + req.Header.Set("Content-Type", "application/json") + resp, err := f.srv.Client().Do(req) + if err != nil { + f.t.Fatalf("post alert: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNoContent { + f.t.Fatalf("post alert status = %d, want 204", resp.StatusCode) + } +} + +// converge runs one full quiescence pass: dispatch/input/controller/Triage +// (replayFixture.convergeAll) followed by delivery rounds until the +// notification ledger is quiescent too. +func (f *historyFixture) converge() { + f.t.Helper() + f.convergeAll(f.l2(), newAcceptingAnalyzer(), &countingAfterCommitter{}, nil) + f.deliver() +} + +// oneRound runs exactly ONE dispatch/input/controller/Triage pass at one +// clock step, then delivers. converge() loops to quiescence, which +// repeatedly advances the clock — enough, on a recovery-pending Situation, +// to run the 120s recovery grace out and terminalize it before the scenario +// meant to. A scenario that needs to observe an intermediate lifecycle +// state (recovery_pending before a refire; a nonterminal owner before an +// artifact is applied) steps it one round at a time instead. +func (f *historyFixture) oneRound() { + f.t.Helper() + f.oneControllerDrainPass(f.l2()) + f.deliver() +} + +// deliver drains the notification ledger with the real +// situation.NotificationWorker against the real *store.Store, honoring this +// fixture's own delivery crash boundary exactly once. +func (f *historyFixture) deliver() { + f.t.Helper() + if f.crashDelivery { + f.crashDelivery = false + f.deliverer.crashAfterCall = true + simulateCrash(f.t, crashBoundaryDeliveryAcknowledgement, func() { + _, _ = f.newNotificationWorker().RunOnce(f.ctx) + }) + f.deliverer.crashAfterCall = false + f.restartAndReplay() + } + for round := 0; round < 8; round++ { + n, err := f.newNotificationWorker().RunOnce(f.ctx) + if err != nil { + f.t.Fatalf("notification worker round: %v", err) + } + if n == 0 { + return + } + } + f.t.Fatal("deliver: notification ledger did not reach quiescence within bounded rounds") +} + +func (f *historyFixture) newNotificationWorker() *situation.NotificationWorker { + return situation.NewNotificationWorker(f.st, f.deliverer, + situation.NotificationWorkerConfig{Owner: f.owner + ":notify"}, f.clock.Now, nil) +} + +// crashControllerCycle claims exactly one due Situation and crashes the +// controller at this fixture's armed boundary while reconciling it, then +// restarts and replays. It is a no-op for the reference run (crashAt == "") +// beyond the restart itself, so both runs see the same clock schedule. +func (f *historyFixture) crashControllerCycle() { + f.t.Helper() + if f.crashAt == "" { + f.restartAndReplay() + return + } + f.clock.Advance(advanceMargin) + claims, err := f.st.ClaimControllerWork(f.ctx, f.owner+":controller", f.clock.Now(), 300*time.Second, 1) + if err != nil { + f.t.Fatalf("claim controller work: %v", err) + } + if len(claims) == 0 { + // Nothing is due at this point in the script; the crash boundary + // simply does not arise here. The restart still happens so the two + // runs stay clock-identical. + f.restartAndReplay() + return + } + faulty := &faultyControllerStore{Store: f.st, armed: f.crashAt} + controller := situation.NewController(faulty, f.l2(), situation.ControllerConfig{}, + f.clock.Now, audit.New(f.st.DB()), nil) + simulateOptionalCrash(f.t, string(f.crashAt), func() { + _ = controller.Reconcile(f.ctx, claims[0]) + }) + f.restartAndReplay() +} + +// simulateOptionalCrash is simulateCrash's scenario-driven sibling. Whether +// a given cycle reaches a given fault site is a property of the SCENARIO, +// not of the code under test: a reuse cycle dispatches no L2 call at all, +// so crashPointRecordAssessmentCall simply never fires in one. Reaching it +// is therefore not required — the cycle completing normally is a legitimate +// outcome, and the scenario then continues as a restart-only run, which +// must still converge to the same canonical history. +// +// What is NOT tolerated is a panic that is not our own sentinel: exactly +// like simulateCrash, a genuine bug in the code under test is re-panicked, +// never swallowed. +func simulateOptionalCrash(t *testing.T, boundary string, fn func()) { + t.Helper() + defer func() { + r := recover() + if r == nil { + return + } + rc, ok := r.(replayCrash) + if !ok || rc.boundary != boundary { + panic(r) + } + }() + fn() +} + +// restartAndReplay is the shared close/reopen/advance + real startup +// recovery sequence both runs perform at every scripted boundary. +func (f *historyFixture) restartAndReplay() { + f.t.Helper() + f.restart() + f.bootReplay() +} + +// ---------------------------------------------------------------------- +// Canonical, ID-independent history state. +// ---------------------------------------------------------------------- + +// canonicalHistory renders every operator-observable Plan 3 record in this +// database as a normalized, comparable text block. Situation IDs are +// replaced by stable ordinals (S1, S2, ...) in creation order and derived +// identities (Transition IDs, intent IDs, idempotency keys, client message +// IDs, Slack timestamps) are omitted, because all of them are digests of +// the Situation ID and so differ by construction between two independent +// databases. Everything else is compared verbatim. +func canonicalHistory(t *testing.T, st *store.Store) string { + t.Helper() + ctx := context.Background() + ordinals := map[string]string{} + sits := canonicalSituationRows(t, st, ctx) + lines := make([]string, 0, 8*len(sits)+1) + + for i, s := range sits { + ordinals[s.id] = fmt.Sprintf("S%d", i+1) + } + for _, s := range sits { + lines = append(lines, fmt.Sprintf("situation %s group=%s lifecycle=%s attention=%s terminal_reason=%s current_sequence=%d root_published=%d", + ordinals[s.id], s.group, s.lifecycle, s.attention, s.terminalReason, s.seq, s.rooted)) + lines = append(lines, canonicalTransitions(t, st, ordinals[s.id], s.id)...) + lines = append(lines, canonicalEpisode(t, st, ordinals[s.id], s.id)...) + lines = append(lines, canonicalIntents(t, st, ordinals[s.id], s.id)...) + lines = append(lines, canonicalArtifactInputs(t, st, ordinals[s.id], s.id)...) + } + lines = append(lines, canonicalInstallationDelivery(t, st)...) + return strings.Join(lines, "\n") +} + +// canonicalSituationRow is one Situation's own normalized header line data. +type canonicalSituationRow struct { + id, group, lifecycle, attention, terminalReason string + seq, rooted int +} + +func canonicalSituationRows(t *testing.T, st *store.Store, ctx context.Context) []canonicalSituationRow { //nolint:revive // ctx after t matches this file's own testing-helper convention. + t.Helper() + rows, err := st.DB().QueryContext(ctx, + `SELECT id, group_key, lifecycle, attention, + COALESCE(terminal_reason,''), current_transition_sequence, + CASE WHEN slack_channel IS NULL THEN 0 ELSE 1 END + FROM situations ORDER BY created_at ASC, id ASC`) + if err != nil { + t.Fatalf("read situations: %v", err) + } + defer func() { _ = rows.Close() }() + var out []canonicalSituationRow + for rows.Next() { + var s canonicalSituationRow + if err := rows.Scan(&s.id, &s.group, &s.lifecycle, &s.attention, &s.terminalReason, &s.seq, &s.rooted); err != nil { + t.Fatalf("scan situation: %v", err) + } + out = append(out, s) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate situations: %v", err) + } + return out +} + +func canonicalTransitions(t *testing.T, st *store.Store, ordinal, situationID string) []string { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT sequence, reason, journal_kind, lifecycle, attention, actor, + COALESCE(interruption_priority,''), drill, + CASE WHEN operator_artifact_input_id IS NULL THEN 0 ELSE 1 END, + COALESCE(json_extract(journal_json,'$.headline'),''), + COALESCE(json_extract(action_contract_json,'$.next_actor'),''), + COALESCE(json_extract(action_contract_json,'$.alertint_action'),''), + COALESCE(json_extract(action_contract_json,'$.operator_action_required'),'') + FROM situation_transitions WHERE situation_id = ? ORDER BY sequence ASC`, situationID) + if err != nil { + t.Fatalf("read transitions: %v", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var seq, drill, artifact int + var reason, kind, lifecycle, attention, actor, priority, headline string + var nextActor, alertintAction, operatorAction string + if err := rows.Scan(&seq, &reason, &kind, &lifecycle, &attention, &actor, &priority, &drill, &artifact, + &headline, &nextActor, &alertintAction, &operatorAction); err != nil { + t.Fatalf("scan transition: %v", err) + } + out = append(out, fmt.Sprintf(" transition %s#%d reason=%s journal=%s lifecycle=%s attention=%s actor=%s priority=%s drill=%d artifact=%d next_actor=%s alertint_action=%s operator_action=%s headline=%q", + ordinal, seq, reason, kind, lifecycle, attention, actor, priority, drill, artifact, + nextActor, alertintAction, operatorAction, headline)) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate transitions: %v", err) + } + return out +} + +func canonicalEpisode(t *testing.T, st *store.Store, ordinal, situationID string) []string { + t.Helper() + row := st.DB().QueryRowContext(context.Background(), + `SELECT version, source_transition_sequence, + json_extract(summary_json,'$.current_attention'), + json_extract(summary_json,'$.peak_attention'), + json_extract(summary_json,'$.investigation_started'), + json_extract(summary_json,'$.recurrence_count'), + COALESCE(json_extract(summary_json,'$.initial_publication_reason'),''), + COALESCE(json_extract(summary_json,'$.latest_material_reason'),''), + COALESCE(json_extract(summary_json,'$.final_outcome'),''), + COALESCE(json_extract(summary_json,'$.remaining_uncertainty'),''), + COALESCE(json_array_length(summary_json,'$.investigation_work'),0), + COALESCE(json_array_length(summary_json,'$.recorded_operator_context'),0) + FROM situation_episode_summaries WHERE situation_id = ?`, situationID) + var version, seq, started, recurrence, work, operatorContext int + var current, peak, initial, latest, outcome, uncertainty string + if err := row.Scan(&version, &seq, ¤t, &peak, &started, &recurrence, + &initial, &latest, &outcome, &uncertainty, &work, &operatorContext); err != nil { + return []string{fmt.Sprintf(" episode %s none", ordinal)} + } + return []string{fmt.Sprintf( + " episode %s version=%d source_sequence=%d attention=%s peak=%s investigation_started=%d recurrence=%d initial=%q latest=%q outcome=%q uncertainty=%q work_entries=%d operator_entries=%d", + ordinal, version, seq, current, peak, started, recurrence, initial, latest, outcome, uncertainty, work, operatorContext)} +} + +// canonicalIntents renders this Situation's LIVE notification intents. A +// superseded intent is deliberately excluded: supersession is precisely the +// mechanism that absorbs a crash between a fenced commit and the moment its +// result was observed (controller_replay_test.go's boundary 9 — the commit +// landed, the process died, the replayed cycle recommitted and its newer +// root projection superseded the older one). A superseded intent never +// reaches Slack, so it is not operator-observable state; what MUST match +// across a crash is the set of effects that actually deliver. +// assertOnlyRootSyncSuperseded separately proves the far stronger property +// the spec actually promises: only a coalescible root projection may ever +// be superseded — an immutable journal entry never is. +func canonicalIntents(t *testing.T, st *store.Store, ordinal, situationID string) []string { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT i.effect_class, i.status, i.main_channel_poke, + COALESCE(i.interruption_priority,''), i.requires_root, + COALESCE(i.summary_version,0), COALESCE(t.sequence,0), + COALESCE(i.delivered_as,'') + FROM notification_intents i + LEFT JOIN situation_transitions t ON t.id = i.transition_id + WHERE i.situation_id = ? AND i.status != 'superseded' + ORDER BY COALESCE(t.sequence,0) ASC, i.effect_class ASC, i.summary_version ASC, i.created_at ASC`, situationID) + if err != nil { + t.Fatalf("read intents: %v", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var poke, requiresRoot, summaryVersion, seq int + var deliveredAs string + var class, status, priority string + if err := rows.Scan(&class, &status, &poke, &priority, &requiresRoot, &summaryVersion, &seq, &deliveredAs); err != nil { + t.Fatalf("scan intent: %v", err) + } + out = append(out, fmt.Sprintf(" intent %s class=%s status=%s poke=%d priority=%s requires_root=%d summary_version=%d transition_sequence=%d delivered_as=%q", + ordinal, class, status, poke, priority, requiresRoot, summaryVersion, seq, deliveredAs)) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate intents: %v", err) + } + return out +} + +func canonicalArtifactInputs(t *testing.T, st *store.Store, ordinal, situationID string) []string { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT o.kind, o.status, o.journal_state, + CASE WHEN o.journaled_transition_id IS NULL THEN 0 ELSE 1 END + FROM situation_input_outbox o + WHERE o.kind IN ('operator_annotation_recorded','captured_verdict_recorded') + AND (o.applied_situation_id = ? + OR o.incident_id IN (SELECT incident_id FROM situation_incidents WHERE situation_id = ?)) + ORDER BY o.kind ASC, o.occurred_at ASC, o.id ASC`, situationID, situationID) + if err != nil { + t.Fatalf("read artifact inputs: %v", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var kind, status, journalState string + var journaled int + if err := rows.Scan(&kind, &status, &journalState, &journaled); err != nil { + t.Fatalf("scan artifact input: %v", err) + } + out = append(out, fmt.Sprintf(" artifact %s kind=%s status=%s journal_state=%s journaled=%d", + ordinal, kind, status, journalState, journaled)) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate artifact inputs: %v", err) + } + return out +} + +func canonicalInstallationDelivery(t *testing.T, st *store.Store) []string { + t.Helper() + ctx := context.Background() + var gaps int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM slack_delivery_gaps`).Scan(&gaps); err != nil { + t.Fatalf("count delivery gaps: %v", err) + } + var openGap int + if err := st.DB().QueryRowContext(ctx, + `SELECT CASE WHEN open_gap_generation IS NULL THEN 0 ELSE 1 END FROM slack_delivery_state WHERE id = 1`).Scan(&openGap); err != nil { + t.Fatalf("read delivery state: %v", err) + } + var systemIntents int + if err := st.DB().QueryRowContext(ctx, + `SELECT COUNT(*) FROM notification_intents WHERE effect_class = 'installation_gap_recovery'`).Scan(&systemIntents); err != nil { + t.Fatalf("count gap recovery intents: %v", err) + } + var streamPending int + if err := st.DB().QueryRowContext(ctx, + `SELECT COUNT(*) FROM situation_transition_stream WHERE status != 'delivered'`).Scan(&streamPending); err != nil { + t.Fatalf("count transition stream rows: %v", err) + } + return []string{fmt.Sprintf("installation gaps=%d open_gap=%d gap_recovery_intents=%d stream_undelivered=%d", + gaps, openGap, systemIntents, streamPending)} +} + +// ---------------------------------------------------------------------- +// Assertion helpers over the canonical projection. +// ---------------------------------------------------------------------- + +// transitionReasons returns every recorded Transition reason across every +// Situation, in (situation creation, sequence) order. +func transitionReasons(t *testing.T, st *store.Store) []string { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT tr.reason FROM situation_transitions tr + JOIN situations s ON s.id = tr.situation_id + ORDER BY s.created_at ASC, s.id ASC, tr.sequence ASC`) + if err != nil { + t.Fatalf("read transition reasons: %v", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var reason string + if err := rows.Scan(&reason); err != nil { + t.Fatalf("scan transition reason: %v", err) + } + out = append(out, reason) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate transition reasons: %v", err) + } + return out +} + +func requireReason(t *testing.T, st *store.Store, want string) { + t.Helper() + got := transitionReasons(t, st) + for _, r := range got { + if r == want { + return + } + } + t.Fatalf("no %q Transition was recorded; reasons = %v", want, got) +} + +func requireJournalKind(t *testing.T, st *store.Store, want string) { + t.Helper() + var n int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM situation_transitions WHERE journal_kind = ?`, want).Scan(&n); err != nil { + t.Fatalf("count journal kind %s: %v", want, err) + } + if n == 0 { + t.Fatalf("no Transition carries journal kind %q", want) + } +} + +// assertSequencesContiguous proves the immutable ledger has no hole and no +// duplicate for any Situation — the single invariant a replayed commit +// would break first if a crash could ever double-apply one. +func assertSequencesContiguous(t *testing.T, st *store.Store) { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT situation_id, sequence FROM situation_transitions ORDER BY situation_id ASC, sequence ASC`) + if err != nil { + t.Fatalf("read transition sequences: %v", err) + } + defer func() { _ = rows.Close() }() + seen := map[string]int{} + for rows.Next() { + var sit string + var seq int + if err := rows.Scan(&sit, &seq); err != nil { + t.Fatalf("scan transition sequence: %v", err) + } + if want := seen[sit] + 1; seq != want { + t.Fatalf("situation %s transition sequence %d, want %d (the ledger has a hole or a duplicate)", sit, seq, want) + } + seen[sit] = seq + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate transition sequences: %v", err) + } +} + +// assertSummaryTracksLedger proves every Situation with a Transition also +// has exactly one Episode summary whose source sequence is the Situation's +// current Transition sequence — the projection fence. +func assertSummaryTracksLedger(t *testing.T, st *store.Store) { + t.Helper() + var bad int + if err := st.DB().QueryRowContext(context.Background(), ` + SELECT COUNT(*) FROM situations s + WHERE s.current_transition_sequence > 0 + AND NOT EXISTS ( + SELECT 1 FROM situation_episode_summaries e + WHERE e.situation_id = s.id + AND e.source_transition_sequence = s.current_transition_sequence)`).Scan(&bad); err != nil { + t.Fatalf("check summary fence: %v", err) + } + if bad != 0 { + t.Fatalf("%d situation(s) have a current Transition with no matching Episode summary version", bad) + } +} + +// assertNoStrandedIntent proves no valid intent was left permanently +// failed: Plan 3's `failed` status is reserved for an invalid durable +// intent, and no scenario in this file ever creates one. +func assertNoStrandedIntent(t *testing.T, st *store.Store) { + t.Helper() + var n int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM notification_intents WHERE status IN ('failed','blocked_configuration')`).Scan(&n); err != nil { + t.Fatalf("count stranded intents: %v", err) + } + if n != 0 { + t.Fatalf("%d notification intent(s) ended failed/blocked_configuration; a valid effect must never exhaust", n) + } +} + +// assertOneRootIntentPerSituation proves a Situation never accumulates two +// simultaneously live root projections: at most one root_sync per Situation +// may be pending at rest, and every superseded one is explicitly marked. +func assertOneRootIntentPerSituation(t *testing.T, st *store.Store) { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT situation_id, COUNT(*) FROM notification_intents + WHERE effect_class = 'root_sync' AND status = 'pending' + GROUP BY situation_id HAVING COUNT(*) > 1`) + if err != nil { + t.Fatalf("read pending roots: %v", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var sit string + var n int + if err := rows.Scan(&sit, &n); err != nil { + t.Fatalf("scan pending roots: %v", err) + } + t.Fatalf("situation %s carries %d pending root_sync intents, want at most 1", sit, n) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate pending roots: %v", err) + } +} + +// assertConverged runs every structural invariant this file's scenarios +// share, then returns the canonical projection for cross-run comparison. +// assertOnlyRootSyncSuperseded proves the supersession rule the spec +// states: only a coalescible current-state root projection may ever be +// superseded. An immutable historical effect — a thread append or a +// broadcast handoff — carries one exact Transition's own journal data and +// must never be dropped because a newer one exists. +func assertOnlyRootSyncSuperseded(t *testing.T, st *store.Store) { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT effect_class, COUNT(*) FROM notification_intents + WHERE status = 'superseded' AND effect_class != 'root_sync' GROUP BY effect_class`) + if err != nil { + t.Fatalf("read superseded intents: %v", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var class string + var n int + if err := rows.Scan(&class, &n); err != nil { + t.Fatalf("scan superseded intent: %v", err) + } + t.Fatalf("%d %s intent(s) were superseded; only a coalescible root_sync may ever be", n, class) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate superseded intents: %v", err) + } +} + +func assertConverged(t *testing.T, st *store.Store) string { + t.Helper() + assertSequencesContiguous(t, st) + assertSummaryTracksLedger(t, st) + assertNoStrandedIntent(t, st) + assertOneRootIntentPerSituation(t, st) + assertOnlyRootSyncSuperseded(t, st) + return canonicalHistory(t, st) +} + +// ---------------------------------------------------------------------- +// Scenario driver. +// ---------------------------------------------------------------------- + +// historyScenario is one scripted episode. run drives the fixture; the +// driver below runs it once with no crash and once with each armed +// boundary, then requires the canonical history to match. +type historyScenario struct { + name string + // run drives one full episode to quiescence. It must call + // f.crashControllerCycle() (or f.deliver() with crashDelivery armed) + // at every point the scenario wants a crash boundary, so both runs see + // the same clock schedule. + run func(f *historyFixture) + // assert runs extra scenario-specific expectations against the + // converged database. Called for both runs. + assert func(t *testing.T, st *store.Store) +} + +// crashBoundaries are the controller crash points every scenario is +// replayed against, plus the reference (restart-only) run. The delivery +// acknowledgement boundary is driven separately, by crashDelivery. +var historyCrashBoundaries = []crashPoint{ + crashPointCommitController, + crashPointCommitControllerAfterCommit, + crashPointRecordAssessmentCall, +} + +func runHistoryScenario(t *testing.T, sc historyScenario) { + t.Helper() + + reference := newHistoryFixture(t, sanitizeOwner(sc.name)+"-ref", "", false) + sc.run(reference) + want := assertConverged(t, reference.st) + if testing.Verbose() { + // The reference projection is the whole point of this fixture; a + // maintainer changing history derivation wants to read it, not + // reverse-engineer it out of a diff. + t.Logf("uninterrupted canonical history for %s:\n%s", sc.name, want) + } + if sc.assert != nil { + sc.assert(t, reference.st) + } + + for _, boundary := range historyCrashBoundaries { + t.Run(string(boundary), func(t *testing.T) { + t.Parallel() + f := newHistoryFixture(t, sanitizeOwner(sc.name)+"-"+sanitizeOwner(string(boundary)), boundary, false) + sc.run(f) + got := assertConverged(t, f.st) + if got != want { + t.Fatalf("canonical history after crashing at %s differs from the uninterrupted run.\n--- uninterrupted ---\n%s\n--- after crash+replay ---\n%s", boundary, want, got) + } + if sc.assert != nil { + sc.assert(t, f.st) + } + }) + } + + t.Run(crashBoundaryDeliveryAcknowledgement, func(t *testing.T) { + t.Parallel() + f := newHistoryFixture(t, sanitizeOwner(sc.name)+"-delivery", "", true) + sc.run(f) + got := assertConverged(t, f.st) + if got != want { + t.Fatalf("canonical history after crashing between the Slack call and its durable acknowledgement differs from the uninterrupted run.\n--- uninterrupted ---\n%s\n--- after crash+replay ---\n%s", got, want) + } + if sc.assert != nil { + sc.assert(t, f.st) + } + }) +} + +func sanitizeOwner(s string) string { + return strings.ReplaceAll(strings.ReplaceAll(s, " ", "-"), "_", "-") +} + +// ---------------------------------------------------------------------- +// TestSituationHistoryRealStoreReplay: the Plan 3 history/delivery replay +// catalog. Each subtest is one scripted episode replayed against every +// crash boundary in historyCrashBoundaries plus the delivery-acknowledgement +// boundary, and compared against its own uninterrupted (restart-only) run. +// ---------------------------------------------------------------------- + +func TestSituationHistoryRealStoreReplay(t *testing.T) { + t.Parallel() + t.Run("first_publication", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioFirstPublication()) }) + t.Run("operator_artifacts", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioOperatorArtifacts()) }) + t.Run("artifact_after_closure", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioArtifactAfterClosure()) }) + t.Run("recovery_refire_recovered", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioRecoveryRefireRecovered()) }) + t.Run("closed_unknown", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioClosedUnknown()) }) + t.Run("deadline_refresh", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioDeadlineRefresh()) }) + t.Run("investigation", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioInvestigation()) }) + t.Run("recurrence_handoff", func(t *testing.T) { t.Parallel(); runHistoryScenario(t, scenarioRecurrenceLineageAndHandoff()) }) +} + +// scenarioFirstPublication: one warranted Situation reaches its first +// authoritative state, which must produce exactly one +// first_authoritative_state Transition, one Episode summary at version 1, +// and one root_sync intent that actually delivers. +func scenarioFirstPublication() historyScenario { + return historyScenario{ + name: "first-publication", + run: func(f *historyFixture) { + f.postAlert("hist-first", "HighLatency", "fp-hist-first", "firing", "warning") + f.drainFoundation() + f.crashControllerCycle() + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + requireReason(t, st, "first_authoritative_state") + requireJournalKind(t, st, "publication") + }, + } +} + +// controllerOnlyDrain drains the controller worker alone — never the +// dispatch or input workers convergeAll/oneControllerDrainPass run first. +// R2's ownership race needs exactly this: an artifact input already +// enqueued in the outbox, a controller cycle that terminalizes the owning +// Situation, and only THEN the input worker that applies the artifact +// against an owner that has meanwhile gone terminal. +func (f *historyFixture) controllerOnlyDrain() { + f.t.Helper() + f.clock.Advance(advanceMargin) + cw := situation.NewControllerWorker(f.st, f.st, f.l2(), situation.ControllerConfig{}, + situation.ControllerWorkerConfig{Owner: f.owner + ":controller", Now: f.clock.Now}, f.clock.Now, + audit.New(f.st.DB()), nil) + if _, err := cw.Drain(f.ctx); err != nil { + f.t.Fatalf("controller-only drain: %v", err) + } + f.assertNoReconcileFailed() +} + +// annotate appends one real operator annotation through the production +// store write path (store.InsertIncidentAnnotation), which is what enqueues +// the operator_annotation_recorded Situation input. +func (f *historyFixture) annotate(incidentID, note string) { + f.t.Helper() + if _, err := f.st.InsertIncidentAnnotation(f.ctx, incidentID, "observation", note); err != nil { + f.t.Fatalf("insert incident annotation: %v", err) + } +} + +// captureVerdict records one real Captured verdict through the production +// store write path (store.PersistVerdictCapture). +func (f *historyFixture) captureVerdict(incidentID, note string) { + f.t.Helper() + if _, _, err := f.st.PersistVerdictCapture(f.ctx, store.VerdictCapture{ + IncidentID: incidentID, + Verdict: "confirmation", + Source: "history-replay-test", + LabelConfidence: 0.9, + ExpectationJSON: `{"expected":"replay"}`, + AnnotationNote: note, + }); err != nil { + f.t.Fatalf("persist verdict capture: %v", err) + } +} + +// scenarioOperatorArtifacts: R1 — two operator artifacts (one annotation, +// one Captured verdict) land between two controller cycles and are both +// journaled, in application order, by the next fenced commit, ahead of any +// controller-state Transition in the same commit. +func scenarioOperatorArtifacts() historyScenario { + return historyScenario{ + name: "operator-artifacts", + run: func(f *historyFixture) { + f.postAlert("hist-artifacts", "HighLatency", "fp-hist-artifacts", "firing", "warning") + f.drainFoundation() + f.converge() + + inc := f.soleIncidentID() + f.annotate(inc, "api-2 is the canary host; rollout paused.") + f.captureVerdict(inc, "confirmed: this is the known canary pattern.") + + f.crashControllerCycle() + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + requireReason(t, st, "operator_artifact_recorded") + requireJournalKind(t, st, "operator_note") + requireJournalKind(t, st, "captured_verdict") + assertArtifactJournalStates(t, st, map[string]int{"journaled": 2}) + assertArtifactTransitionsPrecedeControllerState(t, st) + }, + } +} + +// scenarioArtifactAfterClosure: R2 — an artifact enqueued while its owner +// was nonterminal, but applied only after that owner terminalized, is +// recorded as owner_terminal: never journaled, never lost, and it creates +// no Transition, Episode version, or intent. +func scenarioArtifactAfterClosure() historyScenario { + return historyScenario{ + name: "artifact-after-closure", + run: func(f *historyFixture) { + f.postAlert("hist-closure", "HighLatency", "fp-hist-closure", "firing", "warning") + f.drainFoundation() + // The crash boundary sits on the first publication commit: the + // R2 sequence below must run against an already-converged, + // published Situation, because a crash INSIDE it would (quite + // correctly) roll the terminalization back and let the artifact + // journal normally — a different, already-covered outcome. + f.crashControllerCycle() + f.converge() + + // One round only: converge() would run the 120s recovery grace + // out and terminalize before the artifact is ever enqueued. + f.postAlert("hist-closure", "HighLatency", "fp-hist-closure", "resolved", "warning") + f.oneRound() + assertLifecycle(f.t, f.st, "recovery_pending") + + // The owner is still nonterminal here, so this annotation IS + // enqueued. Nothing applies it yet: controllerOnlyDrain below + // deliberately runs the controller WITHOUT the input worker, so + // the owner terminalizes first — exactly R2's race. + f.annotate(f.soleIncidentID(), "checked the dashboards after recovery.") + f.clock.Advance(10 * time.Minute) + f.controllerOnlyDrain() + assertLifecycle(f.t, f.st, "recovered") + + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + assertArtifactJournalStates(t, st, map[string]int{"owner_terminal": 1}) + var journaled int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM situation_transitions WHERE reason = 'operator_artifact_recorded'`).Scan(&journaled); err != nil { + t.Fatalf("count artifact transitions: %v", err) + } + if journaled != 0 { + t.Fatalf("%d operator_artifact_recorded Transition(s) exist; an artifact applied after closure must never be journaled", journaled) + } + requireReason(t, st, "recovered") + }, + } +} + +// scenarioRecoveryRefireRecovered: the full recovery arc — recovery +// observed, a refire that returns the Situation to active, a second +// recovery observation, and clean grace expiry to recovered. +func scenarioRecoveryRefireRecovered() historyScenario { + return historyScenario{ + name: "recovery-refire-recovered", + run: func(f *historyFixture) { + f.postAlert("hist-recovery", "HighLatency", "fp-hist-recovery", "firing", "warning") + f.drainFoundation() + f.converge() + + // One round per lifecycle step: converge() loops to quiescence, + // which would run the 120s recovery grace out and terminalize + // the Situation before it could ever refire. + f.postAlert("hist-recovery", "HighLatency", "fp-hist-recovery", "resolved", "warning") + f.oneRound() + assertLifecycle(f.t, f.st, "recovery_pending") + + // Refire: the same symptom fires again before grace expires. + f.postAlert("hist-recovery", "HighLatency", "fp-hist-recovery", "firing", "warning") + f.oneRound() + assertLifecycle(f.t, f.st, "active") + + f.crashControllerCycle() + + f.postAlert("hist-recovery", "HighLatency", "fp-hist-recovery", "resolved", "warning") + f.oneRound() + + // Clean grace expiry (default webhook grace is 120s). + f.clock.Advance(10 * time.Minute) + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + requireReason(t, st, "recovery_observed") + requireReason(t, st, "recovery_failed") + requireReason(t, st, "recovered") + requireJournalKind(t, st, "recovery_pending") + requireJournalKind(t, st, "recovery_refired") + requireJournalKind(t, st, "recovered") + assertTerminalIsLast(t, st) + }, + } +} + +// scenarioClosedUnknown: a Situation whose symptoms stopped reporting and +// whose source-aware lifecycle-observation deadline then expired closes as +// closed_unknown, with its terminal reason recorded and no invented +// Monitoring phase. +func scenarioClosedUnknown() historyScenario { + return historyScenario{ + name: "closed-unknown", + run: func(f *historyFixture) { + f.postAlert("hist-unknown", "HighLatency", "fp-hist-unknown", "firing", "warning") + f.drainFoundation() + f.converge() + + // Resolve, then let the observation deadline pass BEFORE any + // controller cycle observes the resolution — the one path that + // reaches closed_unknown straight from active (controller.go's + // resolveLifecycle checks pastDeadline before recovery + // observation). The long duration class's deadline is 7 days. + f.postAlert("hist-unknown", "HighLatency", "fp-hist-unknown", "resolved", "warning") + f.drainFoundation() + f.clock.Advance(8 * 24 * time.Hour) + + f.crashControllerCycle() + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + requireReason(t, st, "closed_unknown") + requireJournalKind(t, st, "closed_unknown") + var reason string + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COALESCE(terminal_reason,'') FROM situations WHERE lifecycle = 'closed_unknown'`).Scan(&reason); err != nil { + t.Fatalf("read terminal reason: %v", err) + } + if reason == "" { + t.Fatal("a closed_unknown Situation carries no terminal reason") + } + assertTerminalIsLast(t, st) + }, + } +} + +// scenarioDeadlineRefresh: R4 — a published root whose delivered promise +// has expired gets exactly one coalescible root_sync refresh from the next +// NON-material reconciliation, and that refresh is never a poke and never a +// thread entry. +func scenarioDeadlineRefresh() historyScenario { + return historyScenario{ + name: "deadline-refresh", + run: func(f *historyFixture) { + f.postAlert("hist-refresh", "HighLatency", "fp-hist-refresh", "firing", "warning") + f.drainFoundation() + f.converge() + + // Let the delivered root's promised update time pass, then run + // a reconciliation that changes nothing material. + f.clock.Advance(45 * time.Minute) + f.crashControllerCycle() + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + var roots, pokes int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COUNT(*), COALESCE(SUM(main_channel_poke),0) FROM notification_intents + WHERE effect_class = 'root_sync'`).Scan(&roots, &pokes); err != nil { + t.Fatalf("count root intents: %v", err) + } + if roots < 2 { + t.Fatalf("root_sync intents = %d, want at least 2 (the first publication plus one R4 deadline refresh)", roots) + } + if pokes != 1 { + t.Fatalf("main-channel pokes across %d root_sync intents = %d, want exactly 1 (only the first publication pokes; a deadline refresh never does)", roots, pokes) + } + // The refresh must not have created a second Transition: a + // non-material reconciliation creates no history at all. + reasons := transitionReasons(t, st) + if len(reasons) != 1 || reasons[0] != "first_authoritative_state" { + t.Fatalf("transition reasons = %v, want exactly [first_authoritative_state]: an R4 refresh must create no Transition", reasons) + } + }, + } +} + +// ---------------------------------------------------------------------- +// Scenario-specific assertion helpers. +// ---------------------------------------------------------------------- + +func assertArtifactJournalStates(t *testing.T, st *store.Store, want map[string]int) { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT journal_state, COUNT(*) FROM situation_input_outbox + WHERE kind IN ('operator_annotation_recorded','captured_verdict_recorded') + GROUP BY journal_state`) + if err != nil { + t.Fatalf("read artifact journal states: %v", err) + } + defer func() { _ = rows.Close() }() + got := map[string]int{} + for rows.Next() { + var state string + var n int + if err := rows.Scan(&state, &n); err != nil { + t.Fatalf("scan artifact journal state: %v", err) + } + got[state] = n + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate artifact journal states: %v", err) + } + if fmt.Sprint(got) != fmt.Sprint(want) { + t.Fatalf("artifact journal states = %v, want %v", got, want) + } +} + +// assertArtifactTransitionsPrecedeControllerState proves R1's ordering +// rule: within one Situation, every operator_artifact_recorded Transition +// committed in the same cycle sits BEFORE the controller-state Transition +// that cycle produced — i.e. no controller-state Transition ever carries a +// lower sequence than an artifact Transition committed after it. +func assertArtifactTransitionsPrecedeControllerState(t *testing.T, st *store.Store) { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), + `SELECT situation_id, sequence, reason, created_at FROM situation_transitions + ORDER BY situation_id ASC, sequence ASC`) + if err != nil { + t.Fatalf("read transitions for ordering: %v", err) + } + defer func() { _ = rows.Close() }() + type rec struct { + seq int + reason string + createdAt string + } + perSituation := map[string][]rec{} + for rows.Next() { + var sit, reason, createdAt string + var seq int + if err := rows.Scan(&sit, &seq, &reason, &createdAt); err != nil { + t.Fatalf("scan transition for ordering: %v", err) + } + perSituation[sit] = append(perSituation[sit], rec{seq, reason, createdAt}) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate transitions for ordering: %v", err) + } + for sit, recs := range perSituation { + byCommit := map[string][]rec{} + for _, r := range recs { + byCommit[r.createdAt] = append(byCommit[r.createdAt], r) + } + for at, commit := range byCommit { + seenControllerState := false + for _, r := range commit { + if r.reason == "operator_artifact_recorded" { + if seenControllerState { + t.Fatalf("situation %s commit at %s: artifact Transition #%d follows a controller-state Transition in the same fenced commit", sit, at, r.seq) + } + continue + } + seenControllerState = true + } + } + } +} + +// assertTerminalIsLast proves a terminal Transition is always the last one +// in its Situation's ledger — a terminal Situation never reopens and never +// journals anything afterwards. +func assertTerminalIsLast(t *testing.T, st *store.Store) { + t.Helper() + var bad int + if err := st.DB().QueryRowContext(context.Background(), ` + SELECT COUNT(*) FROM situation_transitions t + WHERE t.lifecycle IN ('recovered','closed_unknown') + AND EXISTS (SELECT 1 FROM situation_transitions later + WHERE later.situation_id = t.situation_id AND later.sequence > t.sequence)`).Scan(&bad); err != nil { + t.Fatalf("check terminal ordering: %v", err) + } + if bad != 0 { + t.Fatalf("%d terminal Transition(s) are followed by a later Transition in the same Situation", bad) + } +} + +// assertLifecycle pins the NEWEST Situation's lifecycle mid-scenario, so a +// scenario that depends on observing an intermediate state fails loudly at +// the exact step that stopped producing it rather than silently proving a +// weaker property later. +func assertLifecycle(t *testing.T, st *store.Store, want string) { + t.Helper() + var got string + if err := st.DB().QueryRowContext(context.Background(), + `SELECT lifecycle FROM situations ORDER BY created_at DESC, id DESC LIMIT 1`).Scan(&got); err != nil { + t.Fatalf("read situation lifecycle: %v", err) + } + if got != want { + t.Fatalf("situation lifecycle = %q, want %q", got, want) + } +} + +// scenarioInvestigation: a Situation whose member Incident becomes ready +// has Acute Triage requested, which moves the Operator contract to +// run_acute_triage — the durable basis for the root's Investigating phase — +// and then concludes. +func scenarioInvestigation() historyScenario { + return historyScenario{ + name: "investigation", + run: func(f *historyFixture) { + f.postAlert("hist-investigation", "HighLatency", "fp-hist-investigation", "firing", "warning") + f.drainFoundation() + f.converge() + + // A collecting Incident is a clean minimum-member Triage skip; + // a ready one is what makes the controller request Acute Triage. + f.markReady(f.soleIncidentID()) + + f.crashControllerCycle() + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + requireReason(t, st, "investigation_started") + requireJournalKind(t, st, "investigation_started") + var started int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT json_extract(summary_json,'$.investigation_started') FROM situation_episode_summaries`).Scan(&started); err != nil { + t.Fatalf("read investigation_started: %v", err) + } + if started != 1 { + t.Fatal("the Episode summary does not record that investigation started") + } + }, + } +} + +// shortEpisode drives one complete firing -> resolved -> recovered episode +// for group, leaving a terminal Situation behind. Five of them are what +// makes the SIXTH Situation's own recurrence count reach the first +// milestone rung and its elapsed duration eligible for the only non-floor +// Sufficient-reason candidate this build can reach (duration_outlier). +func (f *historyFixture) shortEpisode(group, fingerprint string) { + f.t.Helper() + f.postAlert(group, "HighLatency", fingerprint, "firing", "warning") + f.drainFoundation() + // Close this episode's correlation window explicitly. The Correlator's + // own fixed-window flush runs on the real wall clock, which this + // fixture never advances, so without this every episode's deliveries + // would keep landing in ONE forever-collecting Incident and only one + // Situation would ever exist — the lineage this scenario needs would + // silently never form. + f.markReady(f.newestIncidentID()) + f.converge() + f.postAlert(group, "HighLatency", fingerprint, "resolved", "warning") + f.converge() + assertLifecycle(f.t, f.st, "recovered") +} + +func (f *historyFixture) newestIncidentID() string { + f.t.Helper() + return scalarString(f.t, f.st, `SELECT id FROM incidents ORDER BY created_at DESC, id DESC LIMIT 1`) +} + +// scenarioRecurrenceLineageAndHandoff: five completed episodes for one +// group make the sixth Situation carry a recurrence count at the first +// milestone rung AND make duration_outlier — the only non-floor eligible +// Sufficient reason this build can reach — admissible once it runs long. +// Claiming it while Attention is investigate is exactly what makes the +// controller derive an operator handoff (assessment.go's +// operatorActionRequired), which is the one transition class that earns a +// broadcast reply. +func scenarioRecurrenceLineageAndHandoff() historyScenario { + return historyScenario{ + name: "recurrence-handoff", + run: func(f *historyFixture) { + for i := 0; i < 5; i++ { + f.shortEpisode("hist-lineage", fmt.Sprintf("fp-hist-lineage-%d", i)) + } + + f.postAlert("hist-lineage", "HighLatency", "fp-hist-lineage-live", "firing", "warning") + f.drainFoundation() + f.converge() + + // Run long enough that this Situation's elapsed duration + // exceeds both the p95 and twice the median of the five short + // episodes above, then let the model claim that candidate. + f.clock.Advance(3 * time.Hour) + f.script = l2Script{Attention: situationmodel.AttentionInvestigate, ClaimNonFloorReason: true} + + f.crashControllerCycle() + f.converge() + }, + assert: func(t *testing.T, st *store.Store) { + t.Helper() + assertRecurrenceMilestoneReached(t, st, 5) + assertOperatorHandoffRecorded(t, st) + }, + } +} + +// assertRecurrenceMilestoneReached proves the live Situation's durable +// recurrence count reached want, and that its Episode summary carries it. +// +// Note deliberately NOT asserted here: a `recurrence_milestone` Transition +// REASON. RecurrenceCount is len(prior terminal Situations for this exact +// group), and migration 0014's situations_one_nonterminal_group_idx allows +// at most one nonterminal Situation per group — so no sibling can +// terminalize while this Situation is live, and the count (hence the +// milestone rung in the materiality tuple) is fixed for its whole lifetime. +// The reason itself is therefore unreachable from the natural pipeline in +// this build; history_test.go's own TestBuildTransitionsCatalog covers it +// at the derivation level. What replay must prove here is that the durable +// recurrence count and its milestone rung survive a crash unchanged. +func assertRecurrenceMilestoneReached(t *testing.T, st *store.Store, want int) { + t.Helper() + var got int + if err := st.DB().QueryRowContext(context.Background(), ` + SELECT json_extract(e.summary_json,'$.recurrence_count') + FROM situation_episode_summaries e + JOIN situations s ON s.id = e.situation_id + WHERE s.lifecycle IN ('active','recovery_pending')`).Scan(&got); err != nil { + t.Fatalf("read live episode recurrence count: %v", err) + } + if got != want { + t.Fatalf("live Episode summary recurrence_count = %d, want %d", got, want) + } +} + +// assertOperatorHandoffRecorded proves the handoff actually happened: a +// Transition whose Operator contract names an operator action, and one +// broadcast_handoff intent for it. A handoff is the only journal reply the +// spec lets broadcast to the main channel. +func assertOperatorHandoffRecorded(t *testing.T, st *store.Store) { + t.Helper() + var handoffs int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM situation_transitions + WHERE json_extract(action_contract_json,'$.operator_action_required') IS NOT NULL`).Scan(&handoffs); err != nil { + t.Fatalf("count handoff transitions: %v", err) + } + if handoffs == 0 { + t.Fatal("no Transition records an operator action; the handoff never happened") + } + var broadcasts int + if err := st.DB().QueryRowContext(context.Background(), + `SELECT COUNT(*) FROM notification_intents WHERE effect_class = 'broadcast_handoff'`).Scan(&broadcasts); err != nil { + t.Fatalf("count broadcast intents: %v", err) + } + if broadcasts == 0 { + t.Fatal("no broadcast_handoff intent was created for the operator handoff") + } +} diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go index 4ecef20..44c4b9c 100644 --- a/internal/store/situation_notifications.go +++ b/internal/store/situation_notifications.go @@ -175,10 +175,14 @@ func (s *Store) ClaimNotificationIntents(ctx context.Context, owner string, now placeholders, args := inPlaceholders(ids) updateArgs := append([]any{owner, leaseExpires}, args...) + // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound. + // The annotation sits on its own line ABOVE the statement: gosec attaches + // a #nosec comment to the node it precedes, and a trailing comment on the + // closing line of a multi-line call is not honored. if _, err := tx.ExecContext(ctx, ` UPDATE notification_intents SET claim_owner = ?, lease_expires_at = ?, claim_token = claim_token + 1, attempt_count = attempt_count + 1 - WHERE id IN (`+placeholders+`)`, updateArgs...); err != nil { // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + WHERE id IN (`+placeholders+`)`, updateArgs...); err != nil { return nil, fmt.Errorf("store: claim notification intents: %w", err) } diff --git a/internal/store/situation_transition_stream.go b/internal/store/situation_transition_stream.go index 416052b..14042a1 100644 --- a/internal/store/situation_transition_stream.go +++ b/internal/store/situation_transition_stream.go @@ -106,10 +106,14 @@ func (s *Store) ClaimTransitionStream(ctx context.Context, owner string, now tim placeholders, args := inPlaceholders(ids) updateArgs := append([]any{owner, leaseExpires}, args...) + // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound. + // The annotation sits on its own line ABOVE the statement: gosec attaches + // a #nosec comment to the node it precedes, and a trailing comment on the + // closing line of a multi-line call is not honored. if _, err := tx.ExecContext(ctx, ` UPDATE situation_transition_stream SET lease_owner = ?, lease_expires_at = ?, claim_token = claim_token + 1, attempt_count = attempt_count + 1 - WHERE id IN (`+placeholders+`)`, updateArgs...); err != nil { // #nosec G202 -- placeholders is a fixed "?,?,..." run built from len(ids); every value is bound + WHERE id IN (`+placeholders+`)`, updateArgs...); err != nil { return nil, fmt.Errorf("store: claim transition stream rows: %w", err) } From 4286119c499913294e8767ce54f6b42dd1eae9da Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 10:01:53 +0300 Subject: [PATCH 18/31] docs(config): qualify the recurrence Slack surface by build The `memory` section still said a re-fire edits "the incident's Slack card" unqualified. No Incident-shaped Slack write is reachable on this branch; apply the same released-binary / state-controller qualifier already used for the identical sentence in docs/concepts/architecture.md. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- docs/getting-started/configuration.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index ca5deb2..98433df 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -189,10 +189,11 @@ context for the analysis — at the cost of a slower first finding. Incident memory stops an unchanged, already-analyzed condition from being re-triaged as brand new every time it re-fires. When an alert whose group key matches an already-analyzed incident fires again inside the collapse horizon, -it attaches as a lightweight occurrence — the incident's Slack card edits to -`recurred ×N` — instead of minting a new incident and spending another LLM -call. This is deterministic, free, and always on; there is no enable switch, -only the knobs below. +it attaches as a lightweight occurrence instead of minting a new incident and +spending another LLM call — a released binary edits the Incident card in place +to `recurred ×N`, and on the `state-controller` branch the owning Situation's +own journal carries the recurrence milestone instead. This is deterministic, +free, and always on; there is no enable switch, only the knobs below. | Field | Type | Default | Description | |---|---|---|---| From 6ef26eb6a6239f065fac6f529f1ca89c38c9a99e Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 11:32:38 +0300 Subject: [PATCH 19/31] fix(situation): stop a withheld root stranding the one it replaces Final whole-branch review fixes for Plan 3. A root_sync withheld by the operator's Slack floor superseded nothing, so the pending root it replaced stayed claimable forever against a stale summary version: the Situation never published, every thread_append behind it was permanently unclaimable, and each failed version check was recorded as a Slack dependency failure. - Supersede the pending root_sync whenever any new one is inserted, not only a pending one, so the ledger can never hold two live projections. - Do not withhold a root while an earlier projection is still owed to Slack: the floor gates a new interruption, it does not revoke one already permitted and merely queued (spec.md's ordinary-delay rule). New SnapshotInput/PublicationInput field RootPublicationOwed. - Add DeliveryLocalRetryable so adapter-internal rejections (stale summary version, Store read failure, root not published) retry without moving the Delivery-gap machinery. Only a real Slack wire result does. - Add migration 0019: a partial index for the blocked_configuration count the worker runs every second over an append-only ledger. - Spend the once-per-process configuration reactivation only on an actual reactivation, never on a failed read or a no-op probe. - Correct the docs that claimed an occurrence attach produces a Slack recurrence trace; it produces none. Document the `failed` intent state and its current recovery limits. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EfKEvYSqoFFmL81L1Dsexs Signed-off-by: ernescz --- cmd/alertint/drill.go | 10 +- cmd/alertint/situation_notifications.go | 76 ++++--- cmd/alertint/situation_slack_e2e_test.go | 76 ++++++- docs/concepts/architecture.md | 8 +- docs/concepts/incident-memory.md | 17 +- docs/getting-started/configuration.md | 12 +- docs/integrations/mcp-clients.md | 11 +- docs/notifications/slack.md | 41 +++- internal/situation/controller.go | 1 + internal/situation/notification_plan.go | 20 +- internal/situation/notification_plan_test.go | 33 +++ internal/situation/notification_worker.go | 56 ++++- .../situation/notification_worker_test.go | 96 +++++++++ internal/situation/snapshot.go | 9 + .../0019_notification_blocked_index.sql | 21 ++ ...notification_blocked_index_upgrade_test.go | 196 ++++++++++++++++++ internal/store/situation_controller.go | 23 +- internal/store/situation_history.go | 14 +- internal/store/situation_notifications.go | 11 + .../store/situation_notifications_test.go | 68 ++++++ .../situation_notifications_upgrade_test.go | 6 +- internal/store/store_test.go | 13 +- 22 files changed, 742 insertions(+), 76 deletions(-) create mode 100644 internal/store/migrations/0019_notification_blocked_index.sql create mode 100644 internal/store/notification_blocked_index_upgrade_test.go diff --git a/cmd/alertint/drill.go b/cmd/alertint/drill.go index e15aff3..03dc767 100644 --- a/cmd/alertint/drill.go +++ b/cmd/alertint/drill.go @@ -249,7 +249,7 @@ func (d *drillCmd) run(ctx context.Context) error { return err } } else { - d.printf("fired the rerun; mcp is not usable from here — check the DRILL card edit to \"recurred ×N\".") + d.printf("fired the rerun; mcp is not usable from here — the occurrence count is visible over mcp or in the incidents table (an attach to a live Situation posts nothing to Slack).") } return d.maybeResolve(ctx, run, recvBase, webhookToken) } @@ -499,7 +499,9 @@ func (d *drillCmd) fetchDrillCandidates(ctx context.Context, mcpEndpoint, mcpTok // pollOccurrenceRerun polls the matched incident until its occurrence count // registers the collapsed re-fire, then prints the "recurred ×N" payoff. It -// exits as soon as the count increments; a timeout points at the card edit. +// exits as soon as the count increments; the count is the whole payoff, since +// an attach to a live Situation creates no Transition and therefore no Slack +// effect of any kind. func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToken, incidentID string) error { client := newMCPOneShotClient(mcpEndpoint, mcpToken, d.http) if err := client.initialize(ctx); err != nil { @@ -518,7 +520,7 @@ func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToke Occurrences int `json:"occurrences"` } if json.Unmarshal(raw, &p) == nil && p.Occurrences > 0 { - d.printf("collapsed: incident %s recurred ×%d — no second triage, the existing card edits in place", incidentID, p.Occurrences+1) + d.printf("collapsed: incident %s recurred ×%d — no second triage, and no new Slack message", incidentID, p.Occurrences+1) return nil } } @@ -528,7 +530,7 @@ func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToke } } } - d.printf("the occurrence has not registered yet; check the DRILL card edit to \"recurred ×N\", or re-run with --result %s", incidentID) + d.printf("the occurrence has not registered yet; re-run with --result %s (an attach to a live Situation posts nothing to Slack, so there is no card edit to watch for)", incidentID) return nil } diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go index f7900eb..11a8128 100644 --- a/cmd/alertint/situation_notifications.go +++ b/cmd/alertint/situation_notifications.go @@ -162,11 +162,13 @@ func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.N } view, err := d.store.GetSituationEpisodeView(ctx, *intent.SituationID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) + return situation.NotificationDelivery{}, localDelivery("episode_view_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err)) } if view.Summary.Version != *intent.SummaryVersion { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: intent names summary version %d, current is %d", - *intent.SummaryVersion, view.Summary.Version) + return situation.NotificationDelivery{}, localDelivery("stale_summary_version", + fmt.Errorf("cmd/alertint: situation deliverer: intent names summary version %d, current is %d", + *intent.SummaryVersion, view.Summary.Version)) } recoveryEverObserved, err := d.recoveryEverObserved(ctx, view) @@ -188,7 +190,8 @@ func (d *SituationDeliverer) deliverRootSync(ctx context.Context, intent model.N channel, ts, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + return situation.NotificationDelivery{}, localDelivery("root_coordinates_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err)) } if !ok { res, err := d.api.PostMessage(ctx, slack.PostMessageRequest{ @@ -225,14 +228,17 @@ func (d *SituationDeliverer) deliverThreadAppend(ctx context.Context, intent mod } tr, err := d.store.GetSituationTransition(ctx, *intent.TransitionID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) + return situation.NotificationDelivery{}, localDelivery("transition_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err)) } channel, rootTS, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + return situation.NotificationDelivery{}, localDelivery("root_coordinates_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err)) } if !ok { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) + return situation.NotificationDelivery{}, localDelivery("root_not_published", + fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID)) } rendered, err := slack.RenderSituationJournal(tr) if err != nil { @@ -266,18 +272,22 @@ func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent } tr, err := d.store.GetSituationTransition(ctx, *intent.TransitionID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err) + return situation.NotificationDelivery{}, localDelivery("transition_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load transition: %w", err)) } channel, rootTS, ok, err := d.store.GetSituationRootCoordinates(ctx, *intent.SituationID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err) + return situation.NotificationDelivery{}, localDelivery("root_coordinates_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load root coordinates: %w", err)) } if !ok { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID) + return situation.NotificationDelivery{}, localDelivery("root_not_published", + fmt.Errorf("cmd/alertint: situation deliverer: situation %s has no delivered root to reply under", *intent.SituationID)) } view, err := d.store.GetSituationEpisodeView(ctx, *intent.SituationID) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err) + return situation.NotificationDelivery{}, localDelivery("episode_view_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err)) } current := view.Summary.SourceTransitionSequence == tr.Sequence @@ -320,7 +330,8 @@ func (d *SituationDeliverer) deliverGapRecovery(ctx context.Context, intent mode } gap, err := d.store.GetDeliveryGap(ctx, *intent.GapGeneration) if err != nil { - return situation.NotificationDelivery{}, fmt.Errorf("cmd/alertint: situation deliverer: load delivery gap: %w", err) + return situation.NotificationDelivery{}, localDelivery("delivery_gap_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: load delivery gap: %w", err)) } rendered, err := slack.RenderDeliveryGapNotice(slack.GapNoticeInput{ GapID: gap.ID, @@ -361,7 +372,8 @@ func (d *SituationDeliverer) recoveryEverObserved(ctx context.Context, view stor for page := 0; page < maxLedgerScanPages; page++ { transitions, err := d.store.ListSituationTransitions(ctx, view.Summary.SituationID, cursor, 0) if err != nil { - return false, fmt.Errorf("cmd/alertint: situation deliverer: scan transition ledger: %w", err) + return false, localDelivery("transition_ledger_unavailable", + fmt.Errorf("cmd/alertint: situation deliverer: scan transition ledger: %w", err)) } if len(transitions) == 0 { return false, nil @@ -407,18 +419,33 @@ func invalidDelivery(code string, err error) error { return &deliveryAdapterError{class: situation.DeliveryInvalid, code: code, err: err} } +// localDelivery marks one of this adapter's own errors as a LOCAL +// data-state condition that stopped the attempt before any Slack call was +// made: a Store read that failed, an intent whose summary version is no +// longer current, or a reply whose root is not published yet. It retries +// exactly like any other retryable outcome — none of these proves a +// permanent condition — but it is not a Slack answer, so it must never move +// the dependency-health window or the Delivery-gap machinery. Reporting +// "Slack delivery failing" for a purely local mismatch would name an outage +// that is not happening. +func localDelivery(code string, err error) error { + return &deliveryAdapterError{class: situation.DeliveryLocalRetryable, code: code, err: err} +} + // classifyDeliveryError resolves one failed Deliver call into the closed // situation.DeliveryFailure classification. // // Slack's own typed classification (slack.APIError) passes straight // through: retryable transport/5xx/rate-limit/uncertain outcomes keep their // Retry-After, definite token/scope/channel rejections block on -// configuration, and a malformed payload this build sent is invalid. -// Anything this adapter already proved invalid keeps that verdict. EVERY -// other error — a Store read failure, a stale summary version, a reply -// whose root is not published yet — stays retryable: none of them proves a -// permanent condition, and only a proven one may ever close a durable -// delivery obligation. +// configuration, and a malformed payload this build sent is invalid. That +// is the ONLY source of a Slack-attributed outcome here: internal/notify/ +// slack wraps every wire result — including a failed round trip — in a +// *slack.APIError, so an error that is not one never reached Slack. +// Anything this adapter already classified keeps its verdict. EVERY other +// error is therefore local: it retries (none of them proves a permanent +// condition, and only a proven one may ever close a durable delivery +// obligation) without being attributed to Slack health. func classifyDeliveryError(err error) error { var adapterErr *deliveryAdapterError if errors.As(err, &adapterErr) { @@ -436,7 +463,7 @@ func classifyDeliveryError(err error) error { } return &deliveryAdapterError{class: class, code: apiErr.Code, retryAfter: apiErr.RetryAfter, err: err} } - return &deliveryAdapterError{class: situation.DeliveryRetryable, code: "delivery_failed", err: err} + return &deliveryAdapterError{class: situation.DeliveryLocalRetryable, code: "delivery_failed", err: err} } // ---------------------------------------------------------------------- @@ -563,10 +590,11 @@ type notificationRecovery struct { // restarted"). // // A failed Slack probe is NOT an error: an unreachable or misconfigured -// Slack at boot is an ordinary delay, so steps 5 and 6 are skipped, every -// blocked intent stays durably blocked, and the process still starts. Only a -// genuine Store failure returns an error — and then the caller must not -// start Receivers. +// Slack at boot is an ordinary delay, so step 5 and step 6's REPLAY half are +// skipped — step 6's stale-root supersession still runs — every blocked +// intent stays durably blocked, and the process still starts. Only a genuine +// Store failure returns an error, and then the caller must not start +// Receivers. func (r *notificationRuntime) RecoverAndReactivate(ctx context.Context, now time.Time) (notificationRecovery, error) { var report notificationRecovery diff --git a/cmd/alertint/situation_slack_e2e_test.go b/cmd/alertint/situation_slack_e2e_test.go index c789ee1..83fef10 100644 --- a/cmd/alertint/situation_slack_e2e_test.go +++ b/cmd/alertint/situation_slack_e2e_test.go @@ -246,6 +246,10 @@ type e2eFixture struct { clock *e2eClock slack *fakeSlackServer + // slackFloor is the operator's notify.slack.min_severity floor every + // controller cycle runs under. Empty (the default) is "no floor". + slackFloor model.InterruptionPriority + worker *situation.NotificationWorker l2 *e2eAssessmentClient } @@ -365,7 +369,7 @@ func (f *e2eFixture) seed(groupKey string) string { func (f *e2eFixture) controllerCycle() int { f.t.Helper() f.clock.advance(time.Minute) - cw := situation.NewControllerWorker(f.st, f.st, f.l2, situation.ControllerConfig{}, + cw := situation.NewControllerWorker(f.st, f.st, f.l2, situation.ControllerConfig{SlackFloor: f.slackFloor}, situation.ControllerWorkerConfig{Owner: e2eOwner + ":controller", Now: f.clock.Now}, f.clock.Now, audit.New(f.st.DB()), slog.New(slog.DiscardHandler)) n, err := cw.Drain(f.ctx) @@ -1136,3 +1140,73 @@ func TestSituationSlackE2EFailingRootUpdateKeepsCoordinatesAndRecovers(t *testin } assertJournalRepliesInSequenceOrder(t, f, sitID, publishedTS) } + +// ---------------------------------------------------------------------- +// 8. A root the operator's Slack floor withholds never strands the earlier +// root projection it replaces — and a purely local data-state mismatch +// is never reported as a Slack dependency failure. +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EBelowFloorRootNeverStrandsTheQueuedRootItReplaces(t *testing.T) { + f := newE2EFixture(t) + f.slackFloor = model.InterruptionMedium + f.slack.setScript(alwaysOK) + + // The Situation opens below the operator's floor, so its first root is a + // durably withheld decision and nothing is on screen. + sitID := f.seed("group=e2e-floor-strand") + + // It then escalates above the floor and earns publication. Nothing is + // delivered yet: that root projection is committed and merely queued. + f.l2.steer(model.AttentionInvestigate, false) + f.clock.advance(20 * time.Minute) + if n := f.controllerCycle(); n == 0 { + t.Fatal("no controller work was due; the scenario needs an above-floor escalation") + } + queued := "" + for _, i := range f.intentsOfClass("root_sync") { + if i.Status == "pending" { + if queued != "" { + t.Fatalf("two root projections are pending at once%s", f.intentSummary()) + } + queued = i.ID + } + } + if queued == "" { + t.Fatalf("the escalation above the floor earned no pending root projection%s", f.intentSummary()) + } + + // Cycle two calms the Situation below the floor while that earned root + // is still queued. + f.l2.steer(model.AttentionObserve, false) + f.clock.advance(20 * time.Minute) + if n := f.controllerCycle(); n == 0 { + t.Fatal("no controller work was due; the scenario needs a second material cycle") + } + for _, i := range f.intentsOfClass("root_sync") { + if i.ID == queued && i.Status == "pending" { + t.Fatalf("the earlier root projection is still pending after a newer one replaced it%s", f.intentSummary()) + } + } + + f.deliverUntilQuiet(30) + + // The publication the floor already permitted is not erased by a later + // below-floor commit: spec.md's ordinary-delay rule publishes the + // LATEST informative root rather than dropping an earned, queued one. + if _, ts := f.rootCoordinates(sitID); ts == "" { + t.Fatalf("the Situation never published despite earning publication before the floor engaged%s", f.intentSummary()) + } + if remaining := len(f.pendingBesidesDelivered()); remaining != 0 { + t.Fatalf("%d effect(s) still owed once the queue drained%s", remaining, f.intentSummary()) + } + + // A local data-state mismatch is not a Slack outcome: nothing here may + // register as a Slack dependency failure or open a Delivery gap. + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 0 { + t.Fatalf("delivery gap generations = %d, want 0: Slack answered every call in this scenario", gaps) + } + if failing := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_state WHERE first_failure_at IS NOT NULL`); failing != 0 { + t.Fatal("a Slack-dependency failure window opened although Slack never failed") + } +} diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index 42ed2b5..eb78ff1 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -191,9 +191,11 @@ grouping/dispatch path, and one Slack writer, at a time. Before spending an analysis, **AlertINT** checks whether it has seen this condition before. A re-fire of an already-analyzed group key inside the collapse horizon attaches as an **occurrence** — no second LLM call; a -released binary edits the Incident card in place, and on the -`state-controller` branch the owning Situation's own journal carries the -recurrence milestone instead. A genuinely new incident whose key matches a past +released binary edits the Incident card in place. On the `state-controller` +branch that attach leaves no Slack trace at all: the owning Situation's +recurrence count counts its already-closed predecessors and cannot move while +it is open, so `recurred ×N` shows only on the root of the next Situation the +group opens. A genuinely new incident whose key matches a past analysis gets the prior finding **recalled** into its prompt as a past hypothesis, never as evidence. See [incident memory](incident-memory.md). diff --git a/docs/concepts/incident-memory.md b/docs/concepts/incident-memory.md index 4a356e0..46407c6 100644 --- a/docs/concepts/incident-memory.md +++ b/docs/concepts/incident-memory.md @@ -22,9 +22,13 @@ under the [`memory`](../getting-started/configuration.md#memory) config block. When a firing alert's group key matches an already-analyzed incident and lands inside the **collapse horizon**, AlertINT attaches it as an **occurrence** of -that incident instead of minting a new one and spending another analysis. The -incident's Slack card edits in place — `recurred ×N · last HH:MM` — and a JSON -occurrence line is written to stdout. No second LLM call. +that incident instead of minting a new one and spending another analysis. In a +released binary the incident's Slack card edits in place — `recurred ×N · +last HH:MM`. On the `state-controller` branch nothing is written to Slack at +all for that attach: the owning Situation's recurrence count counts the +Situations that closed before it and cannot move while it is open, so +`recurred ×N` shows only on the root of the next Situation the group opens. +Either way a JSON occurrence line is written to stdout. No second LLM call. The horizon is two clocks: a sliding attach window (default 30 minutes from the last occurrence) and a hard ceiling on the time since the last analysis (default @@ -101,8 +105,11 @@ on every recurrence, and live evidence can retire it. A **confirmation** verdict retires steering — it records that the machine's conclusion is right. Notes written with `alertint_incident_annotate` are context for the next -investigator: they render on the incident's Slack thread (history line plus -a bounded notes list) and in MCP incident reads (`operator_history`), +investigator: in a released binary they render on the incident's Slack thread +(history line plus a bounded notes list); on the `state-controller` branch +they are journalled instead as one attributed `operator_note` entry in the +owning Situation's thread, and only while a nonterminal Situation owns the +incident. Either way they appear in MCP incident reads (`operator_history`), permanent and age-stamped, and never enter the triage prompt or influence recall. diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 98433df..4f3374d 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -191,9 +191,13 @@ re-triaged as brand new every time it re-fires. When an alert whose group key matches an already-analyzed incident fires again inside the collapse horizon, it attaches as a lightweight occurrence instead of minting a new incident and spending another LLM call — a released binary edits the Incident card in place -to `recurred ×N`, and on the `state-controller` branch the owning Situation's -own journal carries the recurrence milestone instead. This is deterministic, -free, and always on; there is no enable switch, only the knobs below. +to `recurred ×N`. On the `state-controller` branch an attach to a Situation +that is still live produces **no Slack trace at all**: a Situation's recurrence +count is the number of *already-closed* Situations for its group and is +therefore fixed for its whole lifetime, so `recurred ×N` appears only on the +root of the **next** Situation that opens for the group, rendered once at that +Situation's first publication. This is deterministic, free, and always on; +there is no enable switch, only the knobs below. | Field | Type | Default | Description | |---|---|---|---| @@ -374,7 +378,7 @@ starts when the aggregate LLM dependency state first becomes `degraded` or | `slack.bot_token_env` | string | — | Required when `slack.enabled: true`. Env var name holding the Slack bot token (`xoxb-…`, requires the `chat:write` scope; no history-read scope is ever requested) | | `slack.channel` | string | — | Required when `slack.enabled: true`. Channel name (e.g. `#alerts`) or ID (e.g. `C1234567890`) | | `slack.min_severity` | string | `low` | The channel-noise floor (`low` \| `medium` \| `high`); stdout always emits regardless. In a released binary it compares against the finding's severity, and an incident suppressed at firing is also suppressed at resolution. On the `state-controller` branch it is the minimum **interruption priority** a *new* main-channel interruption must meet — never alert severity and never a model claim; `critical` always passes, a withheld interruption is durably recorded, and the floor never suppresses Situation state, MCP history, a root edit, or a journal reply. The default posts everything. | -| `slack.recurrence_mode` | string | `change-gated` | How a recurring incident resurfaces in its thread: `change-gated` posts a thread reply only on a real-world change (severity rise, new symptom, faster cadence) or a milestone (×5/×10/×25/×50/×100, then every ×100) — replies stay in the thread, nothing extra is sent to the channel; `off` keeps recurrence to a silent card count-bump. **No effect on the `state-controller` branch** (recurrence is carried by the owning Situation's own journal at the same milestone rungs); the key is still accepted so an existing config keeps loading. See [Slack](../notifications/slack.md) for details. | +| `slack.recurrence_mode` | string | `change-gated` | How a recurring incident resurfaces in its thread: `change-gated` posts a thread reply only on a real-world change (severity rise, new symptom, faster cadence) or a milestone (×5/×10/×25/×50/×100, then every ×100) — replies stay in the thread, nothing extra is sent to the channel; `off` keeps recurrence to a silent card count-bump. **No effect on the `state-controller` branch**: an occurrence attaching to a live Situation posts nothing and edits nothing there, because that Situation's recurrence count cannot change while it is open — `recurred ×N` shows only on the root of the *next* Situation that opens for the group. The key is still accepted so an existing config keeps loading. See [Slack](../notifications/slack.md) for details. | At startup the agent logs one `notifiers ready` line listing the active sinks (and the Slack channel) so you can see where findings will go. Every analysis diff --git a/docs/integrations/mcp-clients.md b/docs/integrations/mcp-clients.md index bafd26b..5b358ad 100644 --- a/docs/integrations/mcp-clients.md +++ b/docs/integrations/mcp-clients.md @@ -151,10 +151,13 @@ restart Windsurf and check **Settings → MCP Servers**: Both feedback writes land whether or not a Situation currently owns the incident. With **no current owner** — none was ever assigned, or the owner had already closed — the write still persists and stays visible through the -incident's own history here and in the audit log; against an already-closed -Situation it appears in that Situation's `artifacts_recorded_after_closure`, -never journalled into the closed episode and never lost. No old Incident -Slack card is resurrected or rewritten either way. +incident's own history here and in the audit log, never journalled into a +closed episode and never lost. A closed Situation's +`artifacts_recorded_after_closure` is narrower than that: it holds only the +race where the owner closed *between* the write being accepted and being +applied. A write against a Situation that was already closed when it landed +is visible through the incident's own history and the audit log only. No old +Incident Slack card is resurrected or rewritten in any of these cases. Read-only toward your systems, always; feedback writes (the last two tools above) land only in AlertINT's own incident state, additive and diff --git a/docs/notifications/slack.md b/docs/notifications/slack.md index a41ecf7..bb290e2 100644 --- a/docs/notifications/slack.md +++ b/docs/notifications/slack.md @@ -94,10 +94,10 @@ A published Situation owns exactly **one** main-channel message — its Journal entries are created for first publication, material investigation changes and conclusions, operator-contract changes, recovery pending, -recovery refire, permitted recurrence milestones, recovery, closure with -uncertainty, and operator write-backs. Routine reconciliation, retry -accounting, and elapsed seconds ticking by create **nothing** — no entry, no -edit, no interruption. +recovery refire, recovery, closure with uncertainty, and operator +write-backs. Routine reconciliation, retry accounting, elapsed seconds +ticking by, and a re-fire attaching to the Situation that is already open +create **nothing** — no entry, no edit, no interruption. ### Orientation @@ -186,6 +186,27 @@ database transaction open across a Slack call. rejection moves the effect to `blocked_configuration`, where it waits durably. Restarting with corrected configuration returns it to pending and it delivers. Nothing is dropped to silence Slack. +- **An effect Slack rejects as impossible is parked in `failed`, visibly.** + `failed` is reserved for a durable intent this build cannot send at all — + a payload Slack rejects outright, or any Slack rejection code this build + does not recognise as retryable or as a configuration problem. The + commonest way to reach it + is to **delete a Situation's root message in Slack by hand**: every later + edit of that root then comes back `message_not_found`, and the root effect + parks in `failed`. It is never retried on its own, and effects that wait + on that root stay pending behind it rather than failing in a chain. + `failed` effects are visible in the durable ledger and over MCP alongside + every other delivery outcome, so a Situation that stops updating in Slack + is diagnosable rather than silent. + + **Honest limitation:** redriving a `failed` effect is a Store operation + (`RedriveFailedNotificationIntent`) with **no operator-facing command in + front of it yet** — recovering one today means direct database access or a + small program against the Store, not a CLI flag. Deleting a Situation's + root message is therefore best avoided: a redrive alone will not bring it + back, because the coordinates AlertINT holds still point at the message + you removed. An operator control surface for redrive, and a specific + recovery for a deleted root, are both deliberately out of this slice. - **Ordering is preserved.** A Situation's root must be durably delivered before any reply is claimable; journal replies deliver in change order; a hand-off's root edit delivers before its broadcast reply; and a stale root @@ -397,10 +418,14 @@ notify: - `off` — recurrence never posts replies; the card's occurrence count still updates in place, silently. -On the integration branch this setting has **no effect**: recurrence is -carried by the owning Situation's own journal at the same milestone rungs, so -a quiet Situation leaves no recurrence trace in Slack at all. The key is still -accepted so an existing `config.yaml` keeps loading. +On the integration branch this setting has **no effect**, and neither does a +re-fire that attaches to a Situation that is already open: it posts no reply +and edits no root. A Situation's recurrence count is the number of *closed* +Situations that preceded it in the same group, and only one Situation per +group can be open at a time — so that count is fixed for the Situation's whole +lifetime. `recurred ×N` therefore appears exactly once per Situation, on the +root of the **next** Situation the group opens, at its first publication. The +key is still accepted so an existing `config.yaml` keeps loading. ## System messages diff --git a/internal/situation/controller.go b/internal/situation/controller.go index 073a8d3..c68fe7d 100644 --- a/internal/situation/controller.go +++ b/internal/situation/controller.go @@ -1137,6 +1137,7 @@ func (c *Controller) buildHistory(claim Claim, basis historyBasis, commit Contro PriorTransition: basis.In.PriorTransition, RootPublished: basis.In.RootPublished, LatestRootSyncVersion: basis.In.LatestRootSyncVersion, + RootPublicationOwed: basis.In.RootPublicationOwed, LastDeliveredRootDeadlineAt: basis.In.LastDeliveredRootDeadlineAt, LastMainChannelPokeAt: basis.In.LastMainChannelPokeAt, SlackFloor: c.cfg.SlackFloor, diff --git a/internal/situation/notification_plan.go b/internal/situation/notification_plan.go index 2440129..6fcf0d7 100644 --- a/internal/situation/notification_plan.go +++ b/internal/situation/notification_plan.go @@ -35,9 +35,13 @@ type PublicationInput struct { PriorTransition *model.Transition // ContractDeadlineAt is the committed nonterminal next_update_at (R4): // the promise the root renders. Nil for a terminal commit. - ContractDeadlineAt *time.Time - RootPublished bool - LatestRootSyncVersion *int + ContractDeadlineAt *time.Time + RootPublished bool + LatestRootSyncVersion *int + // RootPublicationOwed reports whether an earlier root projection for + // this Situation is still owed to Slack (pending, configuration-blocked, + // or failed) — see SnapshotInput.RootPublicationOwed. + RootPublicationOwed bool LastDeliveredRootDeadlineAt *time.Time LastMainChannelPokeAt *time.Time SlackFloor model.InterruptionPriority @@ -92,7 +96,15 @@ func PlanNotificationIntents(in PublicationInput) ([]model.NotificationIntent, e priority := DeriveInterruptionPriority(authority) root.MainChannelPoke = true root.InterruptionPriority = &priority - if !MeetsSlackFloor(priority, in.SlackFloor) { + // The floor gates a NEW interruption. It does not revoke one the + // operator already permitted: while an earlier root projection is + // still owed to Slack, this projection is that same unmet first + // publication re-rendered at the current summary version, and + // spec.md's ordinary-delay rule publishes the latest informative + // root rather than erasing an earned, merely-queued one. Withholding + // here would also strand the projection it replaces — every later + // journal entry waits on a root that would then never deliver. + if !MeetsSlackFloor(priority, in.SlackFloor) && !in.RootPublicationOwed { root.Status = model.IntentWithheld } } diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go index 75c6d5d..38cbff6 100644 --- a/internal/situation/notification_plan_test.go +++ b/internal/situation/notification_plan_test.go @@ -536,6 +536,39 @@ func TestPlanNotificationIntentsFloorWithholdsThePoke(t *testing.T) { } } +// TestPlanNotificationIntentsFloorNeverRevokesAnOwedPublication is the +// other half of the floor rule: it gates a NEW interruption, never one the +// operator already permitted. While an earlier root projection is still +// owed to Slack, this commit's root is that same unmet first publication +// re-rendered at the current summary version — spec.md's ordinary-delay +// rule publishes the latest informative root rather than erasing an +// earned, merely-queued one, and withholding here would strand the +// projection it replaces along with every journal entry waiting on it. +func TestPlanNotificationIntentsFloorNeverRevokesAnOwedPublication(t *testing.T) { + c := hsChange(t) + c.Situation.Attention = model.AttentionObserve + c.Assessment.Attention = model.AttentionObserve + hsUseReason(&c, reasonCodeDurationOutlier) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + in.SlackFloor = model.InterruptionHigh + in.RootPublicationOwed = true + + roots := hsIntentsOfClass(hsPlan(t, in), model.EffectRootSync) + if len(roots) != 1 { + t.Fatalf("got %d root_sync intents, want 1", len(roots)) + } + if roots[0].Status != model.IntentPending { + t.Errorf("status = %q, want pending: the floor may not revoke a publication already owed to Slack", roots[0].Status) + } + if !roots[0].MainChannelPoke || roots[0].InterruptionPriority == nil { + t.Error("the root still records that it is a poke and the priority it was judged against") + } +} + func TestPlanNotificationIntentsWithheldBroadcastKeepsTheJournal(t *testing.T) { c := hsNext(t) hsUseReason(&c, reasonCodeDurationOutlier) diff --git a/internal/situation/notification_worker.go b/internal/situation/notification_worker.go index 7a80ad6..ce644bd 100644 --- a/internal/situation/notification_worker.go +++ b/internal/situation/notification_worker.go @@ -153,6 +153,16 @@ const ( // DeliveryRetryable covers transport failures, 5xx, rate limiting, and // every uncertain outcome. It retries indefinitely. DeliveryRetryable DeliveryErrorClass = "retryable" + // DeliveryLocalRetryable covers an adapter-internal condition that + // stopped the attempt BEFORE any Slack call was made: a Store read + // failure, an intent whose summary version is no longer current, or a + // reply whose root is not published yet. It retries exactly like + // DeliveryRetryable — none of them proves a permanent condition — but + // it is deliberately NOT a Slack outcome, so it never moves the + // dependency-health window or the Delivery-gap machinery. Attributing + // a purely local data-state mismatch to Slack would report an outage + // that is not happening. + DeliveryLocalRetryable DeliveryErrorClass = "local_retryable" // DeliveryConfigurationBlocking covers a definite token/scope/channel/ // authentication rejection. It is durable, keeps its attempts, and // waits for a corrected configuration generation — never exhausted. @@ -163,6 +173,24 @@ const ( DeliveryInvalid DeliveryErrorClass = "invalid" ) +// isSlackOutcome reports whether this class is evidence about the Slack +// dependency itself. Only a real transport/API result is: an invalid +// payload is this build's own bug, and a local data-state condition never +// reached the wire. Neither may open a Delivery gap or hold the +// continuous-failure window open (spec.md's gap lifecycle is defined over +// "Slack delivery failure", not over every failed attempt). +func (c DeliveryErrorClass) isSlackOutcome() bool { + switch c { + case DeliveryInvalid, DeliveryLocalRetryable: + return false + case DeliveryRetryable, DeliveryConfigurationBlocking: + return true + default: + // An unknown class is not proof that Slack answered. + return false + } +} + // DeliveryFailure is the classification a deliverer error may carry. An // error that does not implement it is treated as retryable: this worker // never dead-letters durable operator history on an error it cannot prove @@ -191,6 +219,8 @@ func classifyDeliveryFailure(err error) (DeliveryErrorClass, string, time.Durati return DeliveryConfigurationBlocking, code, 0 case DeliveryInvalid: return DeliveryInvalid, code, 0 + case DeliveryLocalRetryable: + return DeliveryLocalRetryable, code, failure.DeliveryRetryAfter() case DeliveryRetryable: return DeliveryRetryable, code, failure.DeliveryRetryAfter() default: @@ -611,7 +641,7 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState w.mu.Unlock() w.count(func(s *NotificationWorkerStats) { s.ProbeFailures++ }) class, code, _ := classifyDeliveryFailure(err) - if class != DeliveryInvalid { + if class.isSlackOutcome() { w.observeFailure(ctx, state, code, now) } return @@ -657,9 +687,15 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState // instead drive it explicitly at step 5 of spec.md's startup order, before // Receivers start; whichever runs first applies the correction. func (w *NotificationWorker) ReactivateConfiguration(ctx context.Context) (int, error) { - if w.configurationReactivated.Swap(true) { + if w.configurationReactivated.Load() { return 0, nil } + // Read and decide BEFORE consuming the one-shot. Burning it on a failed + // Store read, or on a healthy process that simply had nothing blocked at + // its first probe, would mean a configuration block arising LATER in this + // process's life could never be reactivated without a restart. The flag + // still guards the loop it was added for: an actual reactivation sets it, + // so intents that immediately re-block are not reactivated again here. state, err := w.store.GetSlackDeliveryState(ctx) if err != nil { return 0, fmt.Errorf("situation: notification worker: read slack delivery state: %w", err) @@ -667,6 +703,12 @@ func (w *NotificationWorker) ReactivateConfiguration(ctx context.Context) (int, if state.BlockedConfigurationCount == 0 { return 0, nil } + if w.configurationReactivated.Swap(true) { + // Another goroutine got here first (the worker's own startup probe + // racing Task 9's explicit startup call); whichever won applies the + // correction. + return 0, nil + } generation := state.ConfigurationGeneration + 1 n, err := w.store.ReactivateConfigurationBlocked(ctx, generation, w.now().UTC()) if err != nil { @@ -831,9 +873,11 @@ func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim Notif if stateErr != nil { w.logger.Error("situation: notification worker: read slack delivery state failed", "err", stateErr) } - if class != DeliveryInvalid { - // An invalid payload is this build's own bug, not a Slack outage: - // it must never open a Delivery gap. + if class.isSlackOutcome() { + // Only a real Slack answer moves the dependency-health window. An + // invalid payload is this build's own bug, and a local data-state + // condition never reached the wire; neither may open a Delivery + // gap or report an outage that is not happening. w.observeFailure(ctx, state, code, now) } @@ -844,7 +888,7 @@ func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim Notif var ackErr error switch class { - case DeliveryRetryable: + case DeliveryRetryable, DeliveryLocalRetryable: ackErr = w.retryClaim(ctx, claim, code, retryAfter, now) if ackErr == nil { span.SetAttributes(AttrResultClass.String(DeliverResultRetried)) diff --git a/internal/situation/notification_worker_test.go b/internal/situation/notification_worker_test.go index 6020e1b..0d6ee64 100644 --- a/internal/situation/notification_worker_test.go +++ b/internal/situation/notification_worker_test.go @@ -71,6 +71,7 @@ type nwStore struct { deliverAckErr error heartbeatErr error + stateErr error openGap bool recoverGap string recoverOK bool @@ -148,9 +149,19 @@ func (s *nwStore) CompleteDeliveryGap(_ context.Context, _ time.Time) (string, b func (s *nwStore) GetSlackDeliveryState(context.Context) (SlackDeliveryState, error) { s.mu.Lock() defer s.mu.Unlock() + if s.stateErr != nil { + return SlackDeliveryState{}, s.stateErr + } return s.state, nil } +// setState replaces the health snapshot the next read returns. +func (s *nwStore) setState(mutate func(*nwStore)) { + s.mu.Lock() + defer s.mu.Unlock() + mutate(s) +} + func (s *nwStore) ClaimNotificationIntents(_ context.Context, _ string, _ time.Time, _ time.Duration, _ int) ([]NotificationClaim, error) { s.mu.Lock() defer s.mu.Unlock() @@ -421,6 +432,51 @@ func TestNotificationWorkerBlocksConfigurationAndFailsInvalid(t *testing.T) { }) } +// TestNotificationWorkerLocalRejectionsNeverOpenADeliveryGap proves an +// adapter-internal rejection — a stale summary version, a Store read +// failure, a reply whose root is not published yet — still retries but is +// never attributed to Slack health. No HTTP call was made at all in any of +// those cases, so however long they run they must leave the continuous +// failure window closed and open no Delivery gap: a purely local data-state +// mismatch is not evidence that Slack is down. +func TestNotificationWorkerLocalRejectionsNeverOpenADeliveryGap(t *testing.T) { + clock := &nwClock{at: time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC)} + store := &nwStore{} + const rounds = 4 + for i := 0; i < rounds; i++ { + store.batches = append(store.batches, []NotificationClaim{nwClaim(fmt.Sprintf("intent-%d", i), i+1)}) + } + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{}, nwDeliveryError{class: DeliveryLocalRetryable, code: "stale_summary_version"} + }} + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", Heartbeat: time.Hour, Rand: func() float64 { return 0.5 }, + }, clock.now, nwLogger()) + + // Well past the five-minute continuous-failure threshold. + for i := 0; i < rounds; i++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", i, err) + } + clock.advance(2 * time.Minute) + } + + store.snapshot(func(s *nwStore) { + if len(s.retried) != rounds { + t.Fatalf("retried %d intents, want %d: a local mismatch still retries indefinitely", len(s.retried), rounds) + } + if len(s.failed) != 0 || len(s.blocked) != 0 { + t.Fatalf("failed=%v blocked=%v, want a local mismatch to close no delivery obligation", s.failed, s.blocked) + } + if len(s.failures) != 0 { + t.Fatalf("observed slack failures = %v, want none: no Slack call was ever made", s.failures) + } + if len(s.gapOpens) != 0 { + t.Fatalf("the gap machinery ran %d time(s) on a purely local condition", len(s.gapOpens)) + } + }) +} + // TestNotificationWorkerUncertainSuccessConvergesToOneDeliveredIntent // proves an uncertain external response followed by a successful retry // reuses the identical client message id and converges locally to exactly @@ -827,6 +883,46 @@ func TestNotificationWorkerReactivateConfigurationIsIdempotentForStartup(t *test }) } +// TestNotificationWorkerReactivationOneShotIsSpentOnlyOnRealWork is the +// regression for consuming the once-per-process reactivation on something +// that reactivated nothing. The one-shot exists to stop the +// reactivate/re-block loop above — so it must be spent by an actual +// reactivation, never by a transient Store read failure and never by a +// healthy process that simply had nothing blocked when it first probed. A +// configuration block that arises LATER in the same process must still be +// reactivatable without a restart. +func TestNotificationWorkerReactivationOneShotIsSpentOnlyOnRealWork(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{stateErr: errors.New("store: temporarily unavailable")} + w := nwWorker(store, &nwDeliverer{}, now) + + // A failed read reactivates nothing and spends nothing. + if _, err := w.ReactivateConfiguration(context.Background()); err == nil { + t.Fatal("ReactivateConfiguration returned nil on a failed state read") + } + + // A healthy process with nothing blocked also spends nothing. + store.setState(func(s *nwStore) { s.stateErr = nil; s.state = SlackDeliveryState{ConfigurationGeneration: 4} }) + if n, err := w.ReactivateConfiguration(context.Background()); err != nil || n != 0 { + t.Fatalf("ReactivateConfiguration with nothing blocked = (%d, %v), want (0, nil)", n, err) + } + + // A block that appears later in this same process is still correctable. + store.setState(func(s *nwStore) { s.state.BlockedConfigurationCount = 2 }) + if n, err := w.ReactivateConfiguration(context.Background()); err != nil || n != 1 { + t.Fatalf("ReactivateConfiguration after a later block = (%d, %v), want (1, nil)", n, err) + } + // And the loop guard still holds: the real reactivation spent the one-shot. + if n, err := w.ReactivateConfiguration(context.Background()); err != nil || n != 0 { + t.Fatalf("ReactivateConfiguration after the real one = (%d, %v), want (0, nil)", n, err) + } + store.snapshot(func(s *nwStore) { + if len(s.reactivated) != 1 || s.reactivated[0] != 5 { + t.Fatalf("reactivated with generations %v, want exactly one [5]", s.reactivated) + } + }) +} + // ---------------------------------------------------------------------- // Plan 3 Task 9: the delivery span and the audit trail // ---------------------------------------------------------------------- diff --git a/internal/situation/snapshot.go b/internal/situation/snapshot.go index ae3b055..13f76f2 100644 --- a/internal/situation/snapshot.go +++ b/internal/situation/snapshot.go @@ -63,6 +63,15 @@ type SnapshotInput struct { // none exists — the guard that keeps a commit from planning a root // older than one already queued or delivered. LatestRootSyncVersion *int + // RootPublicationOwed reports whether an earlier root projection for + // this Situation is still an UNMET publication obligation: pending, + // configuration-blocked, or failed. It is false for a Situation whose + // only earlier projections were withheld by the operator's Slack floor + // or already superseded. Publication the floor once permitted is not + // revoked by a later below-floor commit — spec.md's ordinary-delay rule + // publishes the latest informative root rather than erasing an earned, + // merely-queued one. + RootPublicationOwed bool // LastDeliveredRootDeadlineAt is the promised-update instant the most // recently DELIVERED root_sync actually put on screen (R4). A refresh // is due only once this promise has passed and the committed contract diff --git a/internal/store/migrations/0019_notification_blocked_index.sql b/internal/store/migrations/0019_notification_blocked_index.sql new file mode 100644 index 0000000..b4c700d --- /dev/null +++ b/internal/store/migrations/0019_notification_blocked_index.sql @@ -0,0 +1,21 @@ +-- SPDX-License-Identifier: FSL-1.1-ALv2 +-- +-- One index, no schema change: the blocked-configuration backlog count that +-- GetSlackDeliveryState reads on every notification-worker round (once per +-- second for the life of the process, plus once per failed delivery +-- acknowledgement and once at startup). +-- +-- notification_intents is durable delivery history — migration 0018 forbids +-- DELETE on it — and it gains a row on every material commit and every +-- journal/broadcast effect, so it only ever grows. 0018's status-related +-- partial indexes are all predicated on OTHER statuses +-- (notification_intents_claim_idx on `status = 'pending'`, +-- notification_intents_root_dependency_idx on root_sync), so none of them +-- covers `status = 'blocked_configuration'` and that count was a full table +-- scan whose cost grew without bound over an installation's life. +-- +-- This migration adds no table, no column, and no row: it fabricates +-- nothing for any Situation that predates it, exactly like 0018. +-- ---------------------------------------------------------------------- +CREATE INDEX notification_intents_blocked_configuration_idx ON notification_intents(status) + WHERE status = 'blocked_configuration'; diff --git a/internal/store/notification_blocked_index_upgrade_test.go b/internal/store/notification_blocked_index_upgrade_test.go new file mode 100644 index 0000000..5d91aab --- /dev/null +++ b/internal/store/notification_blocked_index_upgrade_test.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + "time" +) + +// ---------------------------------------------------------------------- +// Migration 0019 upgrade test: a populated migration-18 database must gain +// exactly one partial index, keep MaxSchemaVersion honest at 19, pass +// PRAGMA foreign_key_check, fabricate no rows, and actually make the +// blocked-configuration backlog count stop scanning the whole ledger. +// ---------------------------------------------------------------------- + +// seedMigration18BlockedIndexFixture builds a database shaped like the +// schema immediately before 0019 (every embedded migration through 18) and +// seeds one Situation with one Transition and three notification intents — +// one pending, one delivered, one blocked_configuration — so the upgrade is +// exercised over real ledger rows rather than an empty table. +func seedMigration18BlockedIndexFixture(t *testing.T, path string) (situationID string) { + t.Helper() + ctx := context.Background() + + db, err := sql.Open("sqlite", buildDSN(path)) + if err != nil { + t.Fatalf("open fixture db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ) STRICT; + `); err != nil { + t.Fatalf("create schema_migrations: %v", err) + } + + migrations, err := loadMigrations() + if err != nil { + t.Fatalf("load migrations: %v", err) + } + fixture := &Store{db: db} + for _, m := range migrations { + if m.version > 18 { + continue + } + if err := fixture.applyMigration(ctx, m); err != nil { + t.Fatalf("apply migration %d: %v", m.version, err) + } + } + + now := time.Now().UTC() + situationID = "sit-blocked-index" + insertOperationalIncident(ctx, t, fixture, "inc-blocked-index", "group-blocked-index") + if err := insertSituation(ctx, fixture, situationRow{ + id: situationID, groupKey: "group-blocked-index", lifecycle: "active", + }); err != nil { + t.Fatalf("insert situation: %v", err) + } + + transitionID := "tr-blocked-index" + if _, err := db.ExecContext(ctx, ` + INSERT INTO situation_transitions ( + id, situation_id, sequence, input_version, material_fact_hash, lifecycle, attention, + action_contract_json, reason, journal_kind, journal_json, projection_json, + evidence_refs_json, actor, created_at + ) VALUES (?, ?, 1, 1, 'sha256:blocked-index', 'active', 'observe', + '{}', 'first_authoritative_state', 'publication', '{}', '{}', '[]', + 'deterministic_controller', ?)`, + transitionID, situationID, canonicalTime(now)); err != nil { + t.Fatalf("insert transition: %v", err) + } + + rows := []struct{ id, class, status string }{ + {"intent-pending", "thread_append", "pending"}, + {"intent-blocked", "broadcast_handoff", "blocked_configuration"}, + } + for _, r := range rows { + requiresRoot := 1 + poke := 0 + var priority any + if r.class == "broadcast_handoff" { + poke = 1 + priority = "high" + } + if _, err := db.ExecContext(ctx, ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, + requires_root, main_channel_poke, interruption_priority, client_message_id, status, created_at + ) VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?)`, + r.id, "key:"+r.id, r.class, situationID, transitionID, + requiresRoot, poke, priority, "client:"+r.id, r.status, canonicalTime(now)); err != nil { + t.Fatalf("insert %s intent: %v", r.status, err) + } + } + return situationID +} + +// TestNotificationBlockedIndexUpgrade_AddsThePartialIndexAndFabricatesNothing +// is migration 0019's own upgrade test: opening a migration-18 database +// applies it, the head becomes 19, the partial index exists with exactly +// the predicate the blocked-configuration count reads, foreign keys still +// check, and not one ledger row is invented or lost. +func TestNotificationBlockedIndexUpgrade_AddsThePartialIndexAndFabricatesNothing(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migration18-blocked-index.db") + situationID := seedMigration18BlockedIndexFixture(t, path) + + st, err := Open(ctx, path) + if err != nil { + t.Fatalf("open upgraded store: %v", err) + } + defer func() { _ = st.Close() }() + + var applied int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM schema_migrations WHERE version = 19`).Scan(&applied); err != nil { + t.Fatal(err) + } + if applied != 1 { + t.Fatalf("migration 19 applied count = %d, want 1", applied) + } + got, err := MaxSchemaVersion() + if err != nil { + t.Fatalf("MaxSchemaVersion: %v", err) + } + if got != 19 { + t.Fatalf("MaxSchemaVersion = %d, want 19", got) + } + + var indexSQL string + if err := st.DB().QueryRowContext(ctx, ` + SELECT sql FROM sqlite_master + WHERE type = 'index' AND name = 'notification_intents_blocked_configuration_idx'`).Scan(&indexSQL); err != nil { + t.Fatalf("blocked-configuration index missing after upgrade: %v", err) + } + if !strings.Contains(indexSQL, "blocked_configuration") { + t.Fatalf("index is not predicated on the blocked status: %s", indexSQL) + } + + assertNoForeignKeyViolations(ctx, t, st) + + // Nothing invented, nothing lost. + var intents, situations int + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM notification_intents`).Scan(&intents); err != nil { + t.Fatal(err) + } + if intents != 2 { + t.Fatalf("notification_intents count = %d, want the 2 seeded rows", intents) + } + if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM situations WHERE id = ?`, situationID).Scan(&situations); err != nil { + t.Fatal(err) + } + if situations != 1 { + t.Fatalf("seeded situation count = %d, want 1", situations) + } + + // The count the notification worker runs every round now resolves + // through that index instead of scanning the whole durable ledger. + var plan string + rows, err := st.DB().QueryContext(ctx, + `EXPLAIN QUERY PLAN SELECT COUNT(*) FROM notification_intents WHERE status = 'blocked_configuration'`) + if err != nil { + t.Fatalf("explain blocked count: %v", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var id, parent, notUsed int + var detail string + if err := rows.Scan(&id, &parent, ¬Used, &detail); err != nil { + t.Fatalf("scan query plan: %v", err) + } + plan += detail + "\n" + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate query plan: %v", err) + } + if !strings.Contains(plan, "notification_intents_blocked_configuration_idx") { + t.Fatalf("the blocked-configuration count still scans the ledger:\n%s", plan) + } + + // And it still answers truthfully. + state, err := st.GetSlackDeliveryState(ctx) + if err != nil { + t.Fatalf("GetSlackDeliveryState: %v", err) + } + if state.BlockedConfigurationCount != 1 { + t.Fatalf("BlockedConfigurationCount = %d, want the 1 seeded blocked intent", state.BlockedConfigurationCount) + } +} diff --git a/internal/store/situation_controller.go b/internal/store/situation_controller.go index 60370af..8b0e049 100644 --- a/internal/store/situation_controller.go +++ b/internal/store/situation_controller.go @@ -150,6 +150,7 @@ func (s *Store) LoadReconciliationInput(ctx context.Context, claim situation.Cla CurrentSummary: currentSummary, RootPublished: publication.rootPublished, LatestRootSyncVersion: publication.latestRootSyncVersion, + RootPublicationOwed: publication.rootPublicationOwed, LastDeliveredRootDeadlineAt: publication.lastDeliveredRootDeadlineAt, LastMainChannelPokeAt: publication.lastMainChannelPokeAt, PendingArtifacts: artifacts, @@ -162,16 +163,17 @@ func (s *Store) LoadReconciliationInput(ctx context.Context, claim situation.Cla type publicationContext struct { rootPublished bool latestRootSyncVersion *int + rootPublicationOwed bool lastDeliveredRootDeadlineAt *time.Time lastMainChannelPokeAt *time.Time } // loadPublicationContextTx reads the Situation's durable Slack root -// coordinates (migration 0018) plus the three delivery facts publication +// coordinates (migration 0018) plus the four delivery facts publication // planning needs: the newest Episode-summary version any root projection -// already renders, the promised-update instant the last DELIVERED root -// actually put on screen, and when this Situation last poked the main -// channel. +// already renders, whether an earlier root projection is still owed to +// Slack, the promised-update instant the last DELIVERED root actually put +// on screen, and when this Situation last poked the main channel. func loadPublicationContextTx(ctx context.Context, tx *sql.Tx, situationID string) (publicationContext, error) { var out publicationContext var channel, rootTS sql.NullString @@ -196,6 +198,19 @@ func loadPublicationContextTx(ctx context.Context, tx *sql.Tx, situationID strin out.latestRootSyncVersion = &v } + // An earlier root projection still owed to Slack: one this Situation + // has already earned (the operator's floor permitted it) and that has + // not been delivered, superseded, or withheld. `delivered` is excluded + // because it is not owed and rootPublished already reports it. + if err := tx.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' + AND status IN ('pending','blocked_configuration','failed'))`, situationID). + Scan(&out.rootPublicationOwed); err != nil { + return publicationContext{}, fmt.Errorf("store: read owed root projection: %w", err) + } + var deadline sql.NullString err = tx.QueryRowContext(ctx, ` SELECT contract_deadline_at FROM notification_intents diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go index b58bf40..9d48c1a 100644 --- a/internal/store/situation_history.go +++ b/internal/store/situation_history.go @@ -470,12 +470,24 @@ func journalOperatorArtifactTx(ctx context.Context, tx *sql.Tx, inputID, transit // 0018 allows at most one pending, unsuperseded root_sync per Situation, so // the supersession and its replacement have to happen in this one // transaction or not at all (R4). +// +// The new root projection retires the older one WHATEVER status it is +// inserted with, `withheld_by_operator_slack_floor` included. A withheld +// replacement is still the current projection of this Situation's root, and +// the older one is still stale: leaving that older row pending would leave +// a projection of a summary version that is not current any more claimable +// forever — it would fail its pre-I/O version check on every attempt and +// nothing would ever supersede it, stranding the Situation's whole later +// history behind a root that can never deliver. Migration 0018 permits this +// exactly: superseding only requires the older row to be pending and the +// replacement to be recorded, never that the replacement itself be +// deliverable. func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID string, intents []situationmodel.NotificationIntent) error { if len(intents) == 0 { return nil } for _, intent := range intents { - if intent.EffectClass != situationmodel.EffectRootSync || intent.Status != situationmodel.IntentPending { + if intent.EffectClass != situationmodel.EffectRootSync { continue } if err := supersedePendingRootSyncTx(ctx, tx, situationID, intent.ID); err != nil { diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go index 44c4b9c..785d1c7 100644 --- a/internal/store/situation_notifications.go +++ b/internal/store/situation_notifications.go @@ -543,6 +543,17 @@ func (s *Store) RecoverExpiredNotificationClaims(ctx context.Context, now time.T // only way out of `failed`, and the way a failed root releases the // dependent history waiting behind it. // +// It has NO operator-facing caller in this build, deliberately: spec.md +// specifies redrive SEMANTICS ("explicitly redriveable after the underlying +// code or data condition changes"), not a control surface, and Plan 3 adds +// no new operator write surface. Recovering a failed intent today therefore +// means direct Store access. docs/notifications/slack.md states that limit +// plainly under "Delivery: durable intent, indefinite retry, at-least-once" +// rather than leaving it as an undocumented gap; an operator command is +// follow-up work, and should land together with a real recovery for a +// hand-deleted root (a bare redrive cannot fix that case, since the stored +// root coordinates still point at the removed message). +// // A failed ROOT projection shares reactivation's uniqueness hazard: its // Situation may have acquired a newer pending root_sync while this one sat // failed. When the redriven projection is the newer of the two, the pending diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index 3a3156f..b8cf6c3 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -943,3 +943,71 @@ func TestNotificationAckRedriveRefusesBehindANewerRootProjection(t *testing.T) { t.Fatalf("newer root status = %q, want an untouched pending", got.Status) } } + +// TestCommitWithheldRootRetiresThePendingRootItReplaces proves migration +// 0018's "one live root projection per Situation" survives a replacement +// the operator's Slack floor withheld. Superseding only when the newer +// projection is itself deliverable would leave the older, now-stale one +// pending forever: it claims ahead of every reply its Situation owns, fails +// its pre-I/O summary-version check on every attempt, and nothing ever +// supersedes it — stranding the Situation's whole later history behind a +// root that can never deliver. +func TestCommitWithheldRootRetiresThePendingRootItReplaces(t *testing.T) { + st := newTestStore(t) + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, first := snSeedOneCycle(t, st, "group-withheld-root", now) + queued := shIntentOfClass(t, first.History.Intents, situationmodel.EffectRootSync) + if queued.Status != situationmodel.IntentPending { + t.Fatalf("seeded root projection has status %q, want pending", queued.Status) + } + + // A second cycle whose first publication falls below the operator's + // floor. The publication context is supplied directly here, so this + // exercises the LEDGER's own invariant rather than the planner's. + later := now.Add(10 * time.Minute) + shMakeDue(t, st, sitID, later.Add(-time.Minute)) + claim := claimSituation(t, st, sitID, "controller-a", later) + cycle := shPrepare(t, claim, shOperatorContract(later.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, later) + last := first.History.Transitions[len(first.History.Transitions)-1] + cycle.Change.PriorTransition = &last + cycle.Change.PriorSummary = first.History.Summary + cycle.Publish.PriorTransition = &last + cycle.Publish.SlackFloor = situationmodel.InterruptionCritical + concl := *cycle.Change.Projection.Assessment + concl.SufficientReasonCode = "duration_outlier" + cycle.Change.Projection.Assessment = &concl + + commit := shDerive(t, cycle) + withheld := shIntentOfClass(t, commit.History.Intents, situationmodel.EffectRootSync) + if withheld.Status != situationmodel.IntentWithheld { + t.Fatalf("replacement root projection has status %q, want withheld_by_operator_slack_floor", withheld.Status) + } + if err := st.CommitController(context.Background(), claim, commit); err != nil { + t.Fatalf("CommitController: %v", err) + } + + retired := snIntent(t, st, queued.ID) + if retired.Status != situationmodel.IntentSuperseded { + t.Fatalf("the replaced root projection has status %q, want superseded", retired.Status) + } + if retired.ReplacementIntentID == nil || *retired.ReplacementIntentID != withheld.ID { + t.Fatalf("replacement_intent_id = %v, want the withheld replacement %s", retired.ReplacementIntentID, withheld.ID) + } + if retired.SupersessionReason == nil || *retired.SupersessionReason == "" { + t.Fatal("a superseded root projection must record why") + } + if retired.ClaimOwner != nil || retired.LeaseExpiresAt != nil || retired.RetryAt != nil { + t.Fatalf("a superseded root projection keeps no claim or retry state: %+v", retired) + } + + var pending int + if err := st.DB().QueryRowContext(context.Background(), ` + SELECT COUNT(*) FROM notification_intents + WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, sitID).Scan(&pending); err != nil { + t.Fatalf("count pending root projections: %v", err) + } + if pending != 0 { + t.Fatalf("%d root projection(s) still pending behind a withheld replacement", pending) + } +} diff --git a/internal/store/situation_notifications_upgrade_test.go b/internal/store/situation_notifications_upgrade_test.go index 277b2bd..1d888fc 100644 --- a/internal/store/situation_notifications_upgrade_test.go +++ b/internal/store/situation_notifications_upgrade_test.go @@ -121,12 +121,14 @@ func TestSituationNotificationsUpgrade_CreatesStrictTablesAndBumpsSchemaVersion( t.Fatalf("migration 18 applied count = %d, want 1", applied) } + // This test owns "0018 landed", not "0018 is the head": the head number + // itself is TestMaxSchemaVersion's, and later migrations may follow. got, err := MaxSchemaVersion() if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - if got != 18 { - t.Fatalf("MaxSchemaVersion = %d, want 18", got) + if got < 18 { + t.Fatalf("MaxSchemaVersion = %d, want at least 18", got) } for _, table := range []string{"notification_intents", "slack_delivery_gaps", "slack_delivery_state"} { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 331a134..03b9d5f 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -446,12 +446,13 @@ func TestMaxSchemaVersion(t *testing.T) { if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - // 0018_situation_notifications.sql is the newest migration today. Plan 2 - // owns 0015/0016; Plan 3 owns exactly 0017/0018 (spec.md "Persistence - // and migration ownership") and both now land, so the number moves from - // 17 to 18 — this is Plan 3's final schema migration. - if got != 18 { - t.Errorf("MaxSchemaVersion = %d, want 18", got) + // 0019_notification_blocked_index.sql is the newest migration today. + // Plan 2 owns 0015/0016; Plan 3 owns 0017/0018 (spec.md "Persistence and + // migration ownership") plus 0019, which adds one partial index and no + // schema of its own — 0017 and 0018 are final and are never edited, so a + // new migration is the only sanctioned way to add it. + if got != 19 { + t.Errorf("MaxSchemaVersion = %d, want 19", got) } } From 5026c340da34fb5fac8eca7baf2e757ab370cfc5 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 12:38:10 +0300 Subject: [PATCH 20/31] fix(situation): gate first publication on publication authority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Situation that has never earned Slack (no published or owed root) now creates no notification intent unless its authority Transition carries publication authority: an unquieted deterministic floor (also proven by urgent Attention, which validation only ever grants on a floor) or a validated Sufficient reason. Before this, every material Transition planned a root and journal intents and the operator's floor was the only gate, so an observe/no-reason Situation published at the default floor and a quiet Situation closing with uncertainty published at a medium one (review round 1, R1-F1). Once a root is on screen or owed, later commits keep synchronizing it regardless of reason; lifecycle alone grants nothing. The crash-replay harness and the fake-Slack e2e fixture both relied on quiet Situations publishing. Both now open warranted Situations through the one production path a fresh group has — five quiet prior episodes and a duration_outlier claimed at observe Attention — and the replay harness arms its delivery crash boundary only once something is claimable, failing a scenario that never delivers. Under warranted fixtures the harness also exposed that investigation_concluded's actor depended on whether the concluding cycle happened to consult the model; it now records llm only when the conclusion content actually changed. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- cmd/alertint/situation_slack_e2e_test.go | 88 +++++++-- internal/situation/controller_test.go | 9 +- internal/situation/history.go | 31 ++- internal/situation/history_replay_test.go | 150 ++++++++++---- internal/situation/history_test.go | 20 ++ internal/situation/notification_plan.go | 24 ++- internal/situation/notification_plan_test.go | 198 +++++++++++++++++++ internal/situation/priority.go | 32 ++- 8 files changed, 493 insertions(+), 59 deletions(-) diff --git a/cmd/alertint/situation_slack_e2e_test.go b/cmd/alertint/situation_slack_e2e_test.go index 83fef10..66cc9d5 100644 --- a/cmd/alertint/situation_slack_e2e_test.go +++ b/cmd/alertint/situation_slack_e2e_test.go @@ -348,18 +348,47 @@ func e2eFirstNonFloorCandidate(prompt llm.Prompt) (model.ReasonCandidate, bool) return model.ReasonCandidate{}, false } -// seed creates one Situation for groupKey through the real Incident + -// situation-input round trip, then runs one controller cycle so it has a +// seed creates one WARRANTED Situation for groupKey — one that carries +// publication authority at its first controller cycle — through the real +// Incident + situation-input round trip, then runs that cycle so it has a // first authoritative Transition, an Episode summary, and its publication // intents. Returns the Situation ID. +// +// A quiet Situation (observe Attention, no accepted Sufficient reason) has +// state and history but no claim on Slack at all, and a fresh group can +// reach no Sufficient reason on its own: duration_outlier — this build's +// only reachable non-floor candidate — needs at least five comparable +// prior durations. So seed builds exactly that lineage first, lets enough +// time pass that the live Situation's elapsed duration is an outlier, and +// has the model claim it at observe Attention: the lowest interruption +// priority a warranted Situation can publish at. Tests that need the +// quiet case use seedQuiet. func (f *e2eFixture) seed(groupKey string) string { + f.t.Helper() + f.seedPriorTerminalLineage(groupKey, 5) + seedControllerRuntimeSituation(f.t, f.st, groupKey, f.clock.Now()) + // Resolve by INCIDENT, not by group key: the group carries several + // terminal Situations, and seedControllerRuntimeSituation's own + // group-key lookup would return an arbitrary one of them. + sitID := f.situationIDForIncident("inc-" + groupKey) + // 45 minutes: an outlier against the minutes-long priors, still inside + // the "medium" duration class so a later scenario step can cross into + // "long" and change the Assessment basis when it needs the model + // consulted again. + f.clock.advance(45 * time.Minute) + f.l2.steer(model.AttentionObserve, true) + f.controllerCycle() + return sitID +} + +// seedQuiet creates one QUIET Situation for groupKey — observe Attention, +// no Sufficient reason, no prior lineage — and runs its first cycle. Such +// a Situation must leave no Slack trace. +func (f *e2eFixture) seedQuiet(groupKey string) string { f.t.Helper() seedControllerRuntimeSituation(f.t, f.st, groupKey, f.clock.Now()) - // Resolve by INCIDENT, not by group key: a group may already carry - // several terminal Situations (seedPriorTerminalLineage), and - // seedControllerRuntimeSituation's own group-key lookup would then - // return an arbitrary one of them. sitID := f.situationIDForIncident("inc-" + groupKey) + f.l2.steer(model.AttentionObserve, false) f.controllerCycle() return sitID } @@ -954,11 +983,6 @@ func (f *e2eFixture) terminalize(situationID string) { func TestSituationSlackE2EStaleHandoffIsDemotedToADelayedThreadEntry(t *testing.T) { f := newE2EFixture(t) f.slack.setScript(alwaysOK) - // Five completed Situations for this group first: duration_outlier — - // the only non-floor Sufficient-reason candidate this build can reach, - // and therefore the only path to an operator handoff — needs at least - // five comparable prior durations. - f.seedPriorTerminalLineage("group=e2e-stale-handoff", 5) sitID := f.seed("group=e2e-stale-handoff") f.deliverUntilQuiet(12) @@ -1152,13 +1176,16 @@ func TestSituationSlackE2EBelowFloorRootNeverStrandsTheQueuedRootItReplaces(t *t f.slackFloor = model.InterruptionMedium f.slack.setScript(alwaysOK) - // The Situation opens below the operator's floor, so its first root is a - // durably withheld decision and nothing is on screen. + // The Situation opens warranted but below the operator's floor (a + // duration_outlier claimed at observe Attention ranks low), so its + // first root is a durably withheld decision and nothing is on screen. sitID := f.seed("group=e2e-floor-strand") - // It then escalates above the floor and earns publication. Nothing is - // delivered yet: that root projection is committed and merely queued. - f.l2.steer(model.AttentionInvestigate, false) + // It then escalates above the floor: the same reason accepted at + // investigate Attention derives an operator handoff, which ranks high. + // Nothing is delivered yet: that root projection is committed and + // merely queued. + f.l2.steer(model.AttentionInvestigate, true) f.clock.advance(20 * time.Minute) if n := f.controllerCycle(); n == 0 { t.Fatal("no controller work was due; the scenario needs an above-floor escalation") @@ -1210,3 +1237,32 @@ func TestSituationSlackE2EBelowFloorRootNeverStrandsTheQueuedRootItReplaces(t *t t.Fatal("a Slack-dependency failure window opened although Slack never failed") } } + +// ---------------------------------------------------------------------- +// 9. A quiet Situation — observe, no Sufficient reason — leaves no Slack +// trace at all (review round 1, R1-F1). +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EQuietSituationSendsNothing(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysOK) + id := f.seedQuiet("group=e2e-quiet") + view, err := f.st.GetSituationEpisodeView(f.ctx, id) + if err != nil { + t.Fatal(err) + } + if view.SourceTransition.Attention != model.AttentionObserve || view.SourceTransition.Projection.Assessment.SufficientReasonCode != "" { + t.Fatalf("fixture not quiet: %+v", view.SourceTransition) + } + f.deliverUntilQuiet(10) + if calls := f.slack.accepted(); len(calls) != 0 { + t.Fatalf("observe/no-reason Situation sent %d Slack messages", len(calls)) + } + if n := f.scalarInt(`SELECT COUNT(*) FROM notification_intents`); n != 0 { + t.Fatalf("a quiet Situation created %d notification intent(s), want 0 (not even a withheld one)%s", n, f.intentSummary()) + } + // Its history is intact: quiet means no Slack, never no state. + if n := f.scalarInt(`SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ?`, id); n == 0 { + t.Fatal("the quiet Situation has no Transition; silence must not erase history") + } +} diff --git a/internal/situation/controller_test.go b/internal/situation/controller_test.go index ed4b676..b2d6f02 100644 --- a/internal/situation/controller_test.go +++ b/internal/situation/controller_test.go @@ -1674,7 +1674,14 @@ func ctReconcileWith(t *testing.T, in situation.SnapshotInput, claim situation.C // Transition, its folded Episode summary, and the Slack obligations it // warrants, all inside the ONE ControllerCommit Plan 2 already fences. func TestControllerHistoryFirstPublicationCommitsTransitionSummaryAndIntents(t *testing.T) { - commit := ctReconcileOnce(t, ctReuseInput(t), ctBaseClaim(), nil) + // A critical firing delivery makes critical_anchor eligible: the + // deterministic floor is the publication authority a first cycle can + // carry (a quiet observe/no-reason Situation creates no Slack intent). + // The fake client has no scripted answer, so the controller derives the + // deterministic fallback — which selects that floor. + in := ctBaseSnapshotInput() + in.Deliveries = []situation.Delivery{ctDelivery("delivery-1", "incident-1", true, "critical")} + commit := ctReconcileOnce(t, in, ctBaseClaim(), nil) if commit.History == nil { t.Fatal("a first authoritative state must commit durable history") diff --git a/internal/situation/history.go b/internal/situation/history.go index 5711f42..9b6cc77 100644 --- a/internal/situation/history.go +++ b/internal/situation/history.go @@ -384,19 +384,48 @@ func transitionIdentity(situationID string, inputVersion, sequence int, reason m // actually model-validated. Lifecycle, the Operator contract, Triage state, // and recurrence are controller-derived, so they are always // deterministic_controller. Plan 3 never produces `operator_policy`. +// +// investigation_concluded is the one reason with both bases: it is selected +// because the Operator contract's investigation phase ENDED (controller- +// derived, from the Triage result), and it renders the Assessment's +// evidence conclusion. It records `llm` only when that conclusion actually +// changed in this commit. Otherwise the model authored nothing new here — +// whether the concluding cycle happened to consult it or reuse its prior +// answer is a scheduling accident (the crash-replay harness proved a crash +// mid-dispatch flipped the actor), and durable history must not depend on +// scheduling. func controllerActor(change AuthoritativeChange, reason model.TransitionReason) model.TransitionActor { if change.Derivation != model.DerivationModelValidated { return model.ActorDeterministicController } switch reason { //nolint:exhaustive // the default is the point: every other reason records controller-derived lifecycle, contract, Triage, or recurrence state, which is never the model's authorship. case model.ReasonFirstAuthoritativeState, model.ReasonMaterialAssessmentChanged, - model.ReasonAttentionChanged, model.ReasonInvestigationConcluded: + model.ReasonAttentionChanged: return model.ActorLLM + case model.ReasonInvestigationConcluded: + if conclusionContentChanged(change) { + return model.ActorLLM + } + return model.ActorDeterministicController default: return model.ActorDeterministicController } } +// conclusionContentChanged reports whether the Assessment's closed +// conclusion (the model-authored part of the materiality tuple) differs +// from the prior Transition's. A first Transition has nothing to compare +// against and counts as changed. +func conclusionContentChanged(change AuthoritativeChange) bool { + prior := change.PriorTransition + if prior == nil { + return true + } + cur := tupleOf("", "", model.ActionContract{}, change.Projection.Assessment, 0) + prev := tupleOf("", "", model.ActionContract{}, prior.Projection.Assessment, 0) + return canonicalDigest(cur) != canonicalDigest(prev) +} + // controllerEvidenceRefs retains every supporting evidence reference for // this commit: the caller's collected references plus the accepted // Sufficient reason's own, deduplicated and ordered so the durable record diff --git a/internal/situation/history_replay_test.go b/internal/situation/history_replay_test.go index 269444d..caa1313 100644 --- a/internal/situation/history_replay_test.go +++ b/internal/situation/history_replay_test.go @@ -328,6 +328,11 @@ func (f *historyFixture) postJSON(payload map[string]any) { // converge runs one full quiescence pass: dispatch/input/controller/Triage // (replayFixture.convergeAll) followed by delivery rounds until the // notification ledger is quiescent too. +// converge drives the whole pipeline to quiescence at Plan 2's own +// convergeAll definition (a round that dispatched, applied, reconciled, and +// triaged nothing), then drains delivery. Every warranted scenario here +// runs at observe Attention on the slow cadence (openWarrantedSituation), +// so a round with nothing due is reachable. func (f *historyFixture) converge() { f.t.Helper() f.convergeAll(f.l2(), newAcceptingAnalyzer(), &countingAfterCommitter{}, nil) @@ -352,7 +357,11 @@ func (f *historyFixture) oneRound() { // fixture's own delivery crash boundary exactly once. func (f *historyFixture) deliver() { f.t.Helper() - if f.crashDelivery { + // The crash boundary sits BETWEEN a Slack call and its acknowledgement, + // so it can only be exercised by a round that actually makes one. Arm + // it on the first round with a claimable intent; runHistoryScenario + // fails a scenario that never produced one. + if f.crashDelivery && f.claimableIntents() > 0 { f.crashDelivery = false f.deliverer.crashAfterCall = true simulateCrash(f.t, crashBoundaryDeliveryAcknowledgement, func() { @@ -373,6 +382,19 @@ func (f *historyFixture) deliver() { f.t.Fatal("deliver: notification ledger did not reach quiescence within bounded rounds") } +// claimableIntents counts pending intents that are due now and, for a +// reply, whose root is published — what one worker round could deliver. +func (f *historyFixture) claimableIntents() int { + f.t.Helper() + return scalarInt(f.t, f.st, ` + SELECT COUNT(*) FROM notification_intents ni + LEFT JOIN situations s ON s.id = ni.situation_id + WHERE ni.status = 'pending' + AND (ni.retry_at IS NULL OR ni.retry_at <= ?) + AND (ni.requires_root = 0 OR s.slack_root_ts IS NOT NULL)`, + f.clock.Now().UTC().Format(time.RFC3339Nano)) +} + func (f *historyFixture) newNotificationWorker() *situation.NotificationWorker { return situation.NewNotificationWorker(f.st, f.deliverer, situation.NotificationWorkerConfig{Owner: f.owner + ":notify"}, f.clock.Now, nil) @@ -910,6 +932,9 @@ func runHistoryScenario(t *testing.T, sc historyScenario) { t.Parallel() f := newHistoryFixture(t, sanitizeOwner(sc.name)+"-delivery", "", true) sc.run(f) + if f.crashDelivery { + t.Fatal("the scenario never delivered anything, so the delivery crash boundary was never exercised; a scenario proving delivery replay must publish") + } got := assertConverged(t, f.st) if got != want { t.Fatalf("canonical history after crashing between the Slack call and its durable acknowledgement differs from the uninterrupted run.\n--- uninterrupted ---\n%s\n--- after crash+replay ---\n%s", got, want) @@ -951,8 +976,7 @@ func scenarioFirstPublication() historyScenario { return historyScenario{ name: "first-publication", run: func(f *historyFixture) { - f.postAlert("hist-first", "HighLatency", "fp-hist-first", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-first", "fp-hist-first") f.crashControllerCycle() f.converge() }, @@ -1016,11 +1040,10 @@ func scenarioOperatorArtifacts() historyScenario { return historyScenario{ name: "operator-artifacts", run: func(f *historyFixture) { - f.postAlert("hist-artifacts", "HighLatency", "fp-hist-artifacts", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-artifacts", "fp-hist-artifacts") f.converge() - inc := f.soleIncidentID() + inc := f.newestIncidentID() f.annotate(inc, "api-2 is the canary host; rollout paused.") f.captureVerdict(inc, "confirmed: this is the known canary pattern.") @@ -1046,8 +1069,7 @@ func scenarioArtifactAfterClosure() historyScenario { return historyScenario{ name: "artifact-after-closure", run: func(f *historyFixture) { - f.postAlert("hist-closure", "HighLatency", "fp-hist-closure", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-closure", "fp-hist-closure") // The crash boundary sits on the first publication commit: the // R2 sequence below must run against an already-converged, // published Situation, because a crash INSIDE it would (quite @@ -1066,7 +1088,7 @@ func scenarioArtifactAfterClosure() historyScenario { // enqueued. Nothing applies it yet: controllerOnlyDrain below // deliberately runs the controller WITHOUT the input worker, so // the owner terminalizes first — exactly R2's race. - f.annotate(f.soleIncidentID(), "checked the dashboards after recovery.") + f.annotate(f.newestIncidentID(), "checked the dashboards after recovery.") f.clock.Advance(10 * time.Minute) f.controllerOnlyDrain() assertLifecycle(f.t, f.st, "recovered") @@ -1096,8 +1118,7 @@ func scenarioRecoveryRefireRecovered() historyScenario { return historyScenario{ name: "recovery-refire-recovered", run: func(f *historyFixture) { - f.postAlert("hist-recovery", "HighLatency", "fp-hist-recovery", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-recovery", "fp-hist-recovery") f.converge() // One round per lifecycle step: converge() loops to quiescence, @@ -1142,8 +1163,7 @@ func scenarioClosedUnknown() historyScenario { return historyScenario{ name: "closed-unknown", run: func(f *historyFixture) { - f.postAlert("hist-unknown", "HighLatency", "fp-hist-unknown", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-unknown", "fp-hist-unknown") f.converge() // Resolve, then let the observation deadline pass BEFORE any @@ -1183,8 +1203,7 @@ func scenarioDeadlineRefresh() historyScenario { return historyScenario{ name: "deadline-refresh", run: func(f *historyFixture) { - f.postAlert("hist-refresh", "HighLatency", "fp-hist-refresh", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-refresh", "fp-hist-refresh") f.converge() // Let the delivered root's promised update time pass, then run @@ -1208,8 +1227,10 @@ func scenarioDeadlineRefresh() historyScenario { t.Fatalf("main-channel pokes across %d root_sync intents = %d, want exactly 1 (only the first publication pokes; a deadline refresh never does)", roots, pokes) } // The refresh must not have created a second Transition: a - // non-material reconciliation creates no history at all. - reasons := transitionReasons(t, st) + // non-material reconciliation creates no history at all. Scoped + // to the live Situation: its five prior episodes have history + // of their own. + reasons := liveTransitionReasons(t, st) if len(reasons) != 1 || reasons[0] != "first_authoritative_state" { t.Fatalf("transition reasons = %v, want exactly [first_authoritative_state]: an R4 refresh must create no Transition", reasons) } @@ -1221,6 +1242,32 @@ func scenarioDeadlineRefresh() historyScenario { // Scenario-specific assertion helpers. // ---------------------------------------------------------------------- +// liveTransitionReasons is transitionReasons scoped to the newest +// Situation — the one a warranted scenario is about, after its lineage. +func liveTransitionReasons(t *testing.T, st *store.Store) []string { + t.Helper() + rows, err := st.DB().QueryContext(context.Background(), ` + SELECT reason FROM situation_transitions + WHERE situation_id = (SELECT id FROM situations ORDER BY created_at DESC, id DESC LIMIT 1) + ORDER BY sequence ASC`) + if err != nil { + t.Fatalf("read live transition reasons: %v", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var r string + if err := rows.Scan(&r); err != nil { + t.Fatalf("scan live transition reason: %v", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate live transition reasons: %v", err) + } + return out +} + func assertArtifactJournalStates(t *testing.T, st *store.Store, want map[string]int) { t.Helper() rows, err := st.DB().QueryContext(context.Background(), @@ -1341,13 +1388,12 @@ func scenarioInvestigation() historyScenario { return historyScenario{ name: "investigation", run: func(f *historyFixture) { - f.postAlert("hist-investigation", "HighLatency", "fp-hist-investigation", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-investigation", "fp-hist-investigation") f.converge() // A collecting Incident is a clean minimum-member Triage skip; // a ready one is what makes the controller request Acute Triage. - f.markReady(f.soleIncidentID()) + f.markReady(f.newestIncidentID()) f.crashControllerCycle() f.converge() @@ -1358,7 +1404,8 @@ func scenarioInvestigation() historyScenario { requireJournalKind(t, st, "investigation_started") var started int if err := st.DB().QueryRowContext(context.Background(), - `SELECT json_extract(summary_json,'$.investigation_started') FROM situation_episode_summaries`).Scan(&started); err != nil { + `SELECT json_extract(summary_json,'$.investigation_started') FROM situation_episode_summaries + WHERE situation_id = (SELECT id FROM situations ORDER BY created_at DESC, id DESC LIMIT 1)`).Scan(&started); err != nil { t.Fatalf("read investigation_started: %v", err) } if started != 1 { @@ -1390,34 +1437,61 @@ func (f *historyFixture) shortEpisode(group, fingerprint string) { assertLifecycle(f.t, f.st, "recovered") } +// openWarrantedSituation opens the Situation a scenario is about in a state +// that carries publication authority at its FIRST controller cycle. +// +// A quiet Situation — observe Attention, no accepted Sufficient reason — +// has state, Transitions, and MCP history but no claim on Slack at all +// (PlanNotificationIntents, review round 1), and a fresh group can reach no +// Sufficient reason on its own: critical_anchor needs a critical delivery +// (whose urgent fast cadence is exactly advanceMargin, so the R4 deadline +// refresh would re-edit the root on every round of this harness and it +// could never look quiescent), novel_symptom and terminal_uncertainty are +// unreachable in this build, and duration_outlier needs at least five +// comparable prior durations. So this helper builds exactly that lineage — +// five short quiet episodes for group — posts the live alert, and lets +// enough time pass that its elapsed duration is an outlier before the +// first cycle claims duration_outlier at observe Attention: the lowest +// interruption priority a warranted Situation can publish at, on the slow +// cadence that lets the harness converge. +func (f *historyFixture) openWarrantedSituation(group, fingerprint string) { + f.t.Helper() + for i := 0; i < 5; i++ { + f.shortEpisode(group, fmt.Sprintf("%s-prior-%d", fingerprint, i)) + } + f.postAlert(group, "HighLatency", fingerprint, "firing", "warning") + f.drainFoundation() + // 45 minutes: an outlier against the minutes-long priors, but still + // inside the "medium" duration class, so a scenario that later needs a + // fresh L2 judgment (the handoff) can cross into "long" and change the + // Assessment basis — a reuse cycle never consults the model. + f.clock.Advance(45 * time.Minute) + f.script = l2Script{ClaimNonFloorReason: true} +} + func (f *historyFixture) newestIncidentID() string { f.t.Helper() return scalarString(f.t, f.st, `SELECT id FROM incidents ORDER BY created_at DESC, id DESC LIMIT 1`) } -// scenarioRecurrenceLineageAndHandoff: five completed episodes for one -// group make the sixth Situation carry a recurrence count at the first -// milestone rung AND make duration_outlier — the only non-floor eligible -// Sufficient reason this build can reach — admissible once it runs long. -// Claiming it while Attention is investigate is exactly what makes the -// controller derive an operator handoff (assessment.go's -// operatorActionRequired), which is the one transition class that earns a -// broadcast reply. +// scenarioRecurrenceLineageAndHandoff: the five completed episodes every +// warranted scenario opens with make the sixth Situation carry a recurrence +// count at the first milestone rung; claiming duration_outlier while +// Attention is investigate is exactly what makes the controller derive an +// operator handoff (assessment.go's operatorActionRequired), which is the +// one transition class that earns a broadcast reply. func scenarioRecurrenceLineageAndHandoff() historyScenario { return historyScenario{ name: "recurrence-handoff", run: func(f *historyFixture) { - for i := 0; i < 5; i++ { - f.shortEpisode("hist-lineage", fmt.Sprintf("fp-hist-lineage-%d", i)) - } - - f.postAlert("hist-lineage", "HighLatency", "fp-hist-lineage-live", "firing", "warning") - f.drainFoundation() + f.openWarrantedSituation("hist-lineage", "fp-hist-lineage-live") f.converge() - // Run long enough that this Situation's elapsed duration - // exceeds both the p95 and twice the median of the five short - // episodes above, then let the model claim that candidate. + // Cross into the "long" duration class so the next cycle's + // Assessment basis changes and the model is consulted again; + // investigate while still claiming duration_outlier: a + // validated non-floor reason accepted at investigate Attention + // is what derives the operator handoff. f.clock.Advance(3 * time.Hour) f.script = l2Script{Attention: situationmodel.AttentionInvestigate, ClaimNonFloorReason: true} diff --git a/internal/situation/history_test.go b/internal/situation/history_test.go index ac6674f..3b9c2da 100644 --- a/internal/situation/history_test.go +++ b/internal/situation/history_test.go @@ -395,6 +395,8 @@ func TestBuildTransitionsCatalog(t *testing.T) { pokeAllowed: false, }, { + // The contract's investigation phase ended with the model's + // conclusion unchanged: the controller authored this change. name: "investigation concluded", change: func(t *testing.T) AuthoritativeChange { t.Helper() @@ -403,6 +405,24 @@ func TestBuildTransitionsCatalog(t *testing.T) { return c }, reason: model.ReasonInvestigationConcluded, + actor: model.ActorDeterministicController, + journal: model.JournalEvidenceConclusion, + pokeAllowed: false, + }, + { + // The same phase end, but the model-validated Assessment also + // changed its conclusion: that content is the model's. + name: "investigation concluded with a changed conclusion", + change: func(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsNext(t) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + concl := *c.Projection.Assessment + concl.Impact = model.ImpactConfirmed + c.Projection.Assessment = &concl + return c + }, + reason: model.ReasonInvestigationConcluded, actor: model.ActorLLM, journal: model.JournalEvidenceConclusion, pokeAllowed: false, diff --git a/internal/situation/notification_plan.go b/internal/situation/notification_plan.go index 6fcf0d7..a6dfba6 100644 --- a/internal/situation/notification_plan.go +++ b/internal/situation/notification_plan.go @@ -51,7 +51,10 @@ type PublicationInput struct { } // PlanNotificationIntents derives every durable Slack obligation one -// committed reconciliation creates: +// committed reconciliation creates. A commit for a Situation that has +// never earned Slack (no published or owed root) creates NOTHING unless its +// authority Transition carries publication authority (PublicationAuthority); +// otherwise it creates: // // - one coalescible `root_sync` carrying the current Episode-summary // version and the committed contract deadline it renders (R4); @@ -81,9 +84,26 @@ func PlanNotificationIntents(in PublicationInput) ([]model.NotificationIntent, e return planDeadlineRefresh(in) } - out := make([]model.NotificationIntent, 0, len(in.Transitions)+2) authority := in.Transitions[len(in.Transitions)-1] + // Publication authority comes BEFORE the floor. A Situation that has + // never earned Slack — no published root, no root still owed — and + // whose authority Transition carries neither a deterministic floor nor + // a validated Sufficient reason is quiet: it creates no intent at all, + // not a withheld one, because nothing was ever permitted that could be + // withheld (spec.md: "quiet and floor-withheld Situations leave no + // Slack trace"; the priority scale ranks a PERMITTED poke, it does not + // grant permission). Once a root is on screen or owed, every later + // commit keeps synchronizing it regardless of the Sufficient reason: + // the floor "never suppresses ... an already-published root edit", and + // a quiet terminal Transition is exactly the "latest informative + // terminal Episode summary" the ordinary-delay rule posts. + if !in.RootPublished && !in.RootPublicationOwed && !PublicationAuthority(authority) { + return nil, nil + } + + out := make([]model.NotificationIntent, 0, len(in.Transitions)+2) + // The root: a first publication is itself the main-channel poke; every // later synchronization is a silent edit. rootPoke := !in.RootPublished diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go index 38cbff6..4140ff0 100644 --- a/internal/situation/notification_plan_test.go +++ b/internal/situation/notification_plan_test.go @@ -855,3 +855,201 @@ func TestPlanNotificationIntentsRejectsIncoherentInput(t *testing.T) { } }) } + +// ---------------------------------------------------------------------- +// Publication authority precedes the floor (review round 1, R1-F1). +// ---------------------------------------------------------------------- + +// hsQuietChange is a first authoritative state with NO publication +// authority: observe Attention and no accepted Sufficient reason. It has +// state and history, but no claim on Slack. +func hsQuietChange(t *testing.T) AuthoritativeChange { + t.Helper() + c := hsChange(t) + c.Situation.Attention = model.AttentionObserve + concl := *c.Projection.Assessment + concl.SufficientReasonCode = "" + concl.SufficientReasonSummary = "" + c.Projection.Assessment = &concl + c.Assessment = hsAssessment(hsMonitoringContract(c.Now.Add(time.Minute)), concl, model.LifecycleActive, model.AttentionObserve) + return c +} + +func TestPlanNotificationIntentsQuietInitialStateHasNoSlackAuthority(t *testing.T) { + c := hsQuietChange(t) + trs, sum := hsCommitOf(t, c) + if len(trs) != 1 || trs[0].Reason != model.ReasonFirstAuthoritativeState { + t.Fatalf("fixture: want one first_authoritative_state transition, got %+v", trs) + } + if PublicationAuthority(trs[0]) { + t.Fatal("a quiet first state must carry no publication authority") + } + in := hsPub(c, trs, sum) + in.RootPublished = false + in.SlackFloor = model.InterruptionLow + + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("a quiet Situation with no Sufficient reason created %d Slack intents, want 0: %+v", len(got), got) + } +} + +func TestPlanNotificationIntentsQuietSituationStaysSilentAtEveryFloor(t *testing.T) { + for _, floor := range []model.InterruptionPriority{"", model.InterruptionLow, model.InterruptionMedium, + model.InterruptionHigh, model.InterruptionCritical} { + c := hsQuietChange(t) + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + in.SlackFloor = floor + if got := hsPlan(t, in); len(got) != 0 { + t.Errorf("floor %q: a quiet Situation created %d Slack intents, want 0 (the floor ranks a permitted poke; it never grants one)", + floor, len(got)) + } + } +} + +func TestPlanNotificationIntentsQuietArtifactCycleCreatesNothing(t *testing.T) { + // An attributed annotation "grants no new reasoning or publication + // authority" (spec.md "Domain model"): journaled on a quiet, never + // published Situation, it still leaves Slack untouched. + c := hsQuietChange(t) + first := hsOnly(t, c) + sum, err := ProjectEpisode(nil, first) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + c.PriorTransition = &first + c.PriorSummary = &sum + c.Now = c.Now.Add(time.Minute) + c.Situation.InputVersion++ + c.OperatorArtifacts = []OperatorArtifactInput{hsArtifact("a-quiet-1", artifactKindAnnotation, c.Now.Add(-time.Minute))} + + trs, next := hsCommitOf(t, c) + if len(trs) != 1 || trs[0].Reason != model.ReasonOperatorArtifactRecorded { + t.Fatalf("fixture: want exactly the artifact transition, got %+v", trs) + } + in := hsPub(c, trs, next) + in.RootPublished = false + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("an annotation on a quiet Situation created %d Slack intents, want 0: %+v", len(got), got) + } +} + +func TestPlanNotificationIntentsQuietClosureDoesNotGainPublicationAuthority(t *testing.T) { + c := hsQuietChange(t) + trs, sum := hsCommitOf(t, c) + c.PriorTransition = &trs[len(trs)-1] + c.PriorSummary = &sum + c.Now = c.Now.Add(time.Hour) + c.Situation.InputVersion++ + hsClosedUnknown(&c) + + trs, sum = hsCommitOf(t, c) + if len(trs) != 1 || trs[0].Reason != model.ReasonClosedUnknown { + t.Fatalf("fixture: want one closed_unknown transition, got %+v", trs) + } + for _, floor := range []model.InterruptionPriority{"", model.InterruptionMedium} { + in := hsPub(c, trs, sum) + in.RootPublished = false + in.SlackFloor = floor + for _, intent := range hsPlan(t, in) { + if intent.Status == model.IntentPending { + t.Errorf("floor %q: a quiet Situation closing with uncertainty gained a pending %s (priority %v): lifecycle alone is not publication authority", + floor, intent.EffectClass, intent.InterruptionPriority) + } + } + } +} + +func TestPlanNotificationIntentsQuietRecoveryCreatesNothing(t *testing.T) { + c := hsQuietChange(t) + trs, sum := hsCommitOf(t, c) + c.PriorTransition = &trs[len(trs)-1] + c.PriorSummary = &sum + c.Now = c.Now.Add(time.Hour) + c.Situation.InputVersion++ + hsRecovered(&c) + + trs, sum = hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + if got := hsPlan(t, in); len(got) != 0 { + t.Fatalf("a quiet Situation recovering created %d Slack intents, want 0: %+v", len(got), got) + } +} + +func TestPlanNotificationIntentsPublishedRootKeepsSynchronizingWithoutAReason(t *testing.T) { + // Authority gates the FIRST publication only. Once the root is on + // screen, a later commit whose Assessment no longer names a Sufficient + // reason still edits it and still journals — the floor (and the + // authority test) "never suppresses ... an already-published root edit". + c := hsQuietChange(t) + first := hsOnly(t, c) + sum, err := ProjectEpisode(nil, first) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + c.PriorTransition = &first + c.PriorSummary = &sum + c.Now = c.Now.Add(time.Minute) + c.Situation.InputVersion++ + // A quiet material change: Attention moves to investigate with still + // no Sufficient reason. + c.Situation.Attention = model.AttentionInvestigate + c.Assessment.Attention = model.AttentionInvestigate + + trs, next := hsCommitOf(t, c) + if len(trs) != 1 || trs[0].Reason != model.ReasonAttentionChanged { + t.Fatalf("fixture: want one attention_changed transition, got %+v", trs) + } + in := hsPub(c, trs, next) + in.RootPublished = true + + got := hsPlan(t, in) + if roots := hsIntentsOfClass(got, model.EffectRootSync); len(roots) != 1 || roots[0].Status != model.IntentPending { + t.Fatalf("published root: want one pending root_sync, got %+v", roots) + } + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { + t.Errorf("published root: want the journal entry, got %d", len(threads)) + } +} + +func TestPlanNotificationIntentsOwedRootKeepsSynchronizingWithoutAReason(t *testing.T) { + // Likewise for a root that was earned earlier and is merely still + // queued: spec.md's ordinary-delay rule publishes the latest informative + // root rather than erasing an earned one. + c := hsQuietChange(t) + trs, sum := hsCommitOf(t, c) + in := hsPub(c, trs, sum) + in.RootPublished = false + in.RootPublicationOwed = true + + got := hsPlan(t, in) + if roots := hsIntentsOfClass(got, model.EffectRootSync); len(roots) != 1 || roots[0].Status != model.IntentPending { + t.Fatalf("owed root: want one pending root_sync, got %+v", roots) + } +} + +func TestPublicationAuthorityComesFromFloorOrValidatedReason(t *testing.T) { + quiet := hsOnly(t, hsQuietChange(t)) + if PublicationAuthority(quiet) { + t.Error("observe + no reason must carry no authority") + } + critical := hsOnly(t, hsChange(t)) // hsConclusion selects the deterministic critical floor + if !PublicationAuthority(critical) { + t.Error("the deterministic critical floor is publication authority") + } + reasoned := hsChange(t) + hsUseReason(&reasoned, reasonCodeDurationOutlier) + if !PublicationAuthority(hsOnly(t, reasoned)) { + t.Error("a validated non-floor Sufficient reason is publication authority") + } + // A floored proposal whose model omitted the anchor still had its + // Attention raised to urgent by validation; that Attention is the floor. + floored := hsQuietChange(t) + floored.Situation.Attention = model.AttentionUrgent + floored.Assessment.Attention = model.AttentionUrgent + if !PublicationAuthority(hsOnly(t, floored)) { + t.Error("urgent Attention is only reachable through a deterministic floor and is publication authority") + } +} diff --git a/internal/situation/priority.go b/internal/situation/priority.go index 477c4aa..b81869d 100644 --- a/internal/situation/priority.go +++ b/internal/situation/priority.go @@ -2,7 +2,11 @@ package situation -import "github.com/alertint/alertint-agent/internal/situation/model" +import ( + "strings" + + "github.com/alertint/alertint-agent/internal/situation/model" +) // ---------------------------------------------------------------------- // Plan 3 Task 4: deterministic Interruption priority and main-channel poke @@ -59,6 +63,32 @@ func deterministicCriticalFloor(t model.Transition) bool { return t.Projection.Assessment != nil && t.Projection.Assessment.SufficientReasonCode == reasonCodeCriticalAnchor } +// PublicationAuthority reports whether t carries the controller's +// deterministic publication authority: an unquieted deterministic floor or +// a validated Sufficient reason (spec.md "Publication authority and +// Interruption priority": "The controller derives publication authority +// from deterministic floors and a validated Sufficient reason"). A +// Situation whose authority Transition carries neither is quiet — it keeps +// its state, Transitions, and MCP history, but has no claim on Slack at +// all, so it never creates a Slack intent (not even a withheld one). The +// operator's Slack floor is a separate, later question: it ranks a +// PERMITTED new poke against the operator's minimum and can never grant an +// authority the Transition does not have. Lifecycle alone grants none +// either: a quiet Situation closing with uncertainty is still quiet. +// +// Urgent Attention counts as the floor: validateProposalContent rejects +// urgent without a deterministic anchor (`urgent_without_floor`) and raises +// Attention to urgent whenever one is active, but it does not select the +// anchor as the Sufficient reason when the model omitted it — so a floored +// Transition may carry urgent Attention with no reason code, and that +// Attention is itself proof of the proven floor. +func PublicationAuthority(t model.Transition) bool { + if deterministicCriticalFloor(t) || t.Attention == model.AttentionUrgent { + return true + } + return t.Projection.Assessment != nil && strings.TrimSpace(t.Projection.Assessment.SufficientReasonCode) != "" +} + // MeetsSlackFloor reports whether priority is at or above the operator's // configured minimum Interruption priority. An empty floor is "no floor". // critical always passes. From 89bd7d7e30811b74b10853840cd42088a9b441fd Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 12:39:48 +0300 Subject: [PATCH 21/31] fix(runtime): revalidate a handoff against its action, not the latest sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Slack deliverer treated a broadcast_handoff as current only when its Transition was the Episode summary's latest sequence. An attributed annotation necessarily advances that sequence while changing neither Attention, lifecycle, nor the required operator action, so a still-required handoff was demoted to a delayed, "no longer current" thread entry and the warranted interruption was lost (review round 1, R1-F5). situation.HandoffStillCurrent now owns the rule: a poke is current while the Situation is nonterminal, its Attention has not de-escalated, and — for a handoff that asked the operator for something — the current Operator contract still asks for the same thing (the same basis a materially changed required action is judged on). Genuinely changed action, terminal recovery, and de-escalation still demote stale effects. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- cmd/alertint/situation_notifications.go | 5 +- cmd/alertint/situation_notifications_test.go | 43 +++++++++ internal/situation/priority.go | 34 +++++++ internal/situation/priority_test.go | 97 ++++++++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go index 11a8128..81717a4 100644 --- a/cmd/alertint/situation_notifications.go +++ b/cmd/alertint/situation_notifications.go @@ -289,7 +289,10 @@ func (d *SituationDeliverer) deliverBroadcastHandoff(ctx context.Context, intent return situation.NotificationDelivery{}, localDelivery("episode_view_unavailable", fmt.Errorf("cmd/alertint: situation deliverer: load episode view: %w", err)) } - current := view.Summary.SourceTransitionSequence == tr.Sequence + // Revalidate the ACTION, not equality with the latest sequence: an + // annotation advances the sequence without changing what the operator + // is asked to do (situation.HandoffStillCurrent). + current := situation.HandoffStillCurrent(tr, view.Summary) renderTr := tr // a local copy: the durable ledger row is never mutated. if !current { diff --git a/cmd/alertint/situation_notifications_test.go b/cmd/alertint/situation_notifications_test.go index 62cb8bd..9bb58ed 100644 --- a/cmd/alertint/situation_notifications_test.go +++ b/cmd/alertint/situation_notifications_test.go @@ -1048,3 +1048,46 @@ func TestSituationNotificationRuntimeRestartAlonePublishesNothing(t *testing.T) } } } + +// TestSituationDelivererAnnotationDoesNotInvalidateUnchangedHandoff pins +// review round 1, R1-F5: a broadcast_handoff is revalidated against the +// durable ACTION basis, never against equality with the latest Transition +// sequence. An attributed annotation necessarily advances the sequence +// while changing neither Attention, lifecycle, nor the required operator +// action — it must not demote a still-current handoff. +func TestSituationDelivererAnnotationDoesNotInvalidateUnchangedHandoff(t *testing.T) { + occurred := sdMustTime(t, "2026-09-05T09:15:00Z") + now := sdMustTime(t, "2026-09-05T10:00:00Z") + action := model.OperatorActionInvestigateSituation + contract := model.ActionContract{ + NextActor: model.NextActorOperator, OperatorActionRequired: &action, + NextUpdateAt: sdTimePtr(now.Add(time.Hour)), NextUpdateOn: []model.NextUpdateOn{model.NextUpdateOnMaterialInput}, + } + handoff := sdTransition(6, model.LifecycleActive, contract, model.ReasonOperatorContractChanged, model.JournalOperatorContractChanged, + model.JournalData{Headline: "Operator action required: investigate_situation", OccurredAt: occurred}, + model.ProjectionFacts{EffectiveStartedAt: occurred, EffectiveStartedAtBasis: model.SourceTimeBasisSourcePayload}, occurred) + note := handoff + note.ID = "annotation-transition" + note.Sequence = 7 + note.Reason = model.ReasonOperatorArtifactRecorded + note.JournalKind = model.JournalOperatorNote + note.Actor = model.ActorAttributedOperator + artifact := "annotation-input" + note.OperatorArtifactInputID = &artifact + note.Journal = model.JournalData{Headline: "Added context", AttributedActor: "operator", OccurredAt: now} + + fs := &fakeDelivererStore{ + episode: store.SituationEpisodeView{Summary: sdSummary(7, contract, occurred, now), SourceTransition: note}, + transitions: map[string]model.Transition{handoff.ID: handoff}, + rootOK: true, rootChannel: "C-existing", rootTS: "50.5", + } + api := &fakeSlackAPI{} + d := NewSituationDeliverer(fs, api, "C-default", func() time.Time { return now }) + got, err := d.Deliver(context.Background(), sdThreadIntent(model.EffectBroadcastHandoff, handoff.ID, handoff.Sequence, now)) + if err != nil { + t.Fatal(err) + } + if got.DeliveredAs != "broadcast" { + t.Fatalf("annotation with identical Attention/lifecycle/operator action suppressed a still-current handoff: delivered_as=%s", got.DeliveredAs) + } +} diff --git a/internal/situation/priority.go b/internal/situation/priority.go index b81869d..fb8d964 100644 --- a/internal/situation/priority.go +++ b/internal/situation/priority.go @@ -164,3 +164,37 @@ func ClassifyPoke(prior *model.Transition, t model.Transition) PokeClass { return PokeNone } } + +// HandoffStillCurrent answers the deliverer's revalidation question for one +// broadcast_handoff (spec.md "Recovery replay": "if its requested action is +// no longer current, the same Transition is delivered as a non-broadcast +// entry marked delayed and no longer current"). It compares the poke's +// durable INTERRUPTION BASIS with the Situation's current authoritative +// state — never mere equality with the latest Transition sequence, which +// an attributed annotation advances without steering anything (review +// round 1, R1-F5): +// +// - a terminal Situation has no current interruption; +// - de-escalated Attention demotes the poke it followed; +// - a handoff that asked the operator for something stays current only +// while the current Operator contract still asks for the same thing +// (operatorContractTuple, the same basis PokeRequiredActionChanged is +// judged on); an escalation poke that asked for no operator action +// (newly urgent Attention, newly crossed criticality) is current while +// its Attention still holds. +// +// A summary that does not yet include the handoff cannot confirm it and +// counts as not current. +func HandoffStillCurrent(handoff model.Transition, summary model.EpisodeSummary) bool { + if summary.SourceTransitionSequence < handoff.Sequence || summary.TerminalAt != nil { + return false + } + if attentionRank(summary.CurrentAttention) < attentionRank(handoff.Attention) { + return false + } + if handoff.ActionContract.OperatorActionRequired == nil { + return true + } + return summary.ActionContract.OperatorActionRequired != nil && + operatorContractTuple(summary.ActionContract) == operatorContractTuple(handoff.ActionContract) +} diff --git a/internal/situation/priority_test.go b/internal/situation/priority_test.go index a130e98..2d7fe59 100644 --- a/internal/situation/priority_test.go +++ b/internal/situation/priority_test.go @@ -304,3 +304,100 @@ func TestInterruptionPriorityArtifactsNeverPoke(t *testing.T) { t.Errorf("artifact poke class = %q, want %q", class, PokeNone) } } + +// ---------------------------------------------------------------------- +// Handoff revalidation (review round 1, R1-F5). +// ---------------------------------------------------------------------- + +func TestHandoffStillCurrentComparesTheActionBasis(t *testing.T) { + now := hsNow(t) + handoffChange := hsChange(t) + handoffChange.Assessment.ActionContract = hsOperatorContract(now.Add(time.Minute)) + handoff := hsOnly(t, handoffChange) + if handoff.ActionContract.OperatorActionRequired == nil { + t.Fatal("fixture: the handoff transition carries no operator action") + } + base, err := ProjectEpisode(nil, handoff) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + + // An annotation folded after the handoff: sequence advanced, nothing + // steered. + annotated := base + annotated.Version++ + annotated.SourceTransitionSequence = handoff.Sequence + 1 + annotated.RecordedOperatorContext = []string{"operator: checked the deploy log"} + if !HandoffStillCurrent(handoff, annotated) { + t.Error("an annotation must not cancel a still-required handoff") + } + + // The same action refreshed with a new deadline is the same action. + refreshed := base + refreshed.SourceTransitionSequence = handoff.Sequence + 1 + refreshed.ActionContract.NextUpdateAt = timePtr(now.Add(time.Hour)) + if !HandoffStillCurrent(handoff, refreshed) { + t.Error("a refreshed deadline does not change the requested action") + } + + // The action was withdrawn: AlertINT took the Situation back. + withdrawn := base + withdrawn.SourceTransitionSequence = handoff.Sequence + 1 + withdrawn.ActionContract = hsMonitoringContract(now.Add(time.Minute)) + if HandoffStillCurrent(handoff, withdrawn) { + t.Error("a handoff whose action was withdrawn is no longer current") + } + + // The Situation terminalized. + terminal := base + terminal.SourceTransitionSequence = handoff.Sequence + 1 + terminal.TerminalAt = timePtr(now.Add(time.Hour)) + terminal.ActionContract = hsTerminalContract() + if HandoffStillCurrent(handoff, terminal) { + t.Error("a terminal Situation has no current interruption") + } + + // Attention de-escalated below the handoff's while the action text + // happened to survive. + calmer := base + calmer.SourceTransitionSequence = handoff.Sequence + 1 + calmer.CurrentAttention = model.AttentionObserve + if handoff.Attention == model.AttentionObserve { + t.Fatal("fixture: the handoff is already at observe Attention") + } + if HandoffStillCurrent(handoff, calmer) { + t.Error("de-escalated Attention demotes the poke it followed") + } + + // A summary that predates the handoff cannot confirm it. + stale := base + stale.SourceTransitionSequence = handoff.Sequence - 1 + if HandoffStillCurrent(handoff, stale) { + t.Error("a summary older than the handoff cannot confirm it as current") + } +} + +func TestHandoffStillCurrentEscalationPokeNeedsNoOperatorAction(t *testing.T) { + // hsChange's conclusion is the deterministic critical floor: an urgent + // escalation poke with no operator action. + c := hsChange(t) + c.Situation.Attention = model.AttentionUrgent + c.Assessment.Attention = model.AttentionUrgent + poke := hsOnly(t, c) + if poke.ActionContract.OperatorActionRequired != nil { + t.Fatal("fixture: the escalation poke must carry no operator action") + } + sum, err := ProjectEpisode(nil, poke) + if err != nil { + t.Fatalf("ProjectEpisode: %v", err) + } + later := sum + later.SourceTransitionSequence = poke.Sequence + 1 + if !HandoffStillCurrent(poke, later) { + t.Error("an escalation poke stays current while its Attention holds") + } + later.CurrentAttention = model.AttentionInvestigate + if HandoffStillCurrent(poke, later) { + t.Error("an escalation poke is demoted once Attention de-escalates") + } +} From 5cd373fb4d10525b9f12485c89900e01fe68ef6f Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 12:44:16 +0300 Subject: [PATCH 22/31] fix(store): keep a superseded first post's coordinates as the Situation root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A concurrent controller commit supersedes even a claimed first-root projection and clears its claim. When Slack had already accepted that post, its acknowledgement returned ErrNotificationIntentSuperseded before the returned coordinates were persisted, so the replacement projection found no root and posted a second one — with no lost response and no crash (review round 1, R1-F3). Supersession keeps the claim token. MarkNotificationDelivered now honors a root_sync acknowledgement whose row was superseded under that exact token: it records the accepted post as the Situation's root when none exists yet (never overwriting an existing root — a superseded edit moved nothing), and still reports the intent as superseded. A stale holder whose lease was reclaimed carries an older token and writes nothing. The worker's heartbeat no longer cancels an in-flight delivery on supersession — no other holder exists, and canceling would manufacture an uncertain outcome — so the attempt finishes and is acknowledged; the superseded audit event now carries the delivered coordinates. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- cmd/alertint/situation_slack_e2e_test.go | 42 ++++++++ internal/situation/notification_worker.go | 26 ++++- .../situation/notification_worker_test.go | 74 +++++++++++++- internal/store/situation_history.go | 18 ++-- internal/store/situation_notifications.go | 53 +++++++++- .../store/situation_notifications_test.go | 97 +++++++++++++++++++ 6 files changed, 296 insertions(+), 14 deletions(-) diff --git a/cmd/alertint/situation_slack_e2e_test.go b/cmd/alertint/situation_slack_e2e_test.go index 66cc9d5..89e4eb8 100644 --- a/cmd/alertint/situation_slack_e2e_test.go +++ b/cmd/alertint/situation_slack_e2e_test.go @@ -1266,3 +1266,45 @@ func TestSituationSlackE2EQuietSituationSendsNothing(t *testing.T) { t.Fatal("the quiet Situation has no Transition; silence must not erase history") } } + +// ---------------------------------------------------------------------- +// 11. A first root post that Slack accepted is never discarded because a +// concurrent controller commit superseded its projection mid-flight: +// the replacement edits that root, it does not post a second one +// (review round 1, R1-F3). +// ---------------------------------------------------------------------- + +func TestSituationSlackE2ESupersededSuccessfulFirstPostDoesNotCreateSecondRoot(t *testing.T) { + f := newE2EFixture(t) + f.seed("group=e2e-post-race") + changed := false + f.slack.setScript(func(method string, call *e2eSlackCall) e2eSlackReply { + if method == "chat.postMessage" && call.ThreadTS == "" && !changed { + changed = true + // Slack has accepted this post; before the worker can + // acknowledge it, a material commit replaces its projection. + f.l2.steer(model.AttentionInvestigate, true) + f.clock.advance(3 * time.Hour) + if f.controllerCycle() == 0 { + t.Error("fixture did not run concurrent controller commit") + } + } + return e2eSlackReply{} + }) + f.deliverUntilQuiet(20) + if !changed { + t.Fatal("the fixture never intercepted a first root post") + } + roots := 0 + for _, c := range f.slack.accepted() { + if c.Method == "chat.postMessage" && c.ThreadTS == "" { + roots++ + } + } + if roots != 1 { + t.Fatalf("%d successful root posts with no transport uncertainty or process crash; in-flight supersession discarded the first coordinates%s", roots, f.intentSummary()) + } + if remaining := len(f.pendingBesidesDelivered()); remaining != 0 { + t.Fatalf("%d effect(s) still owed once the queue drained%s", remaining, f.intentSummary()) + } +} diff --git a/internal/situation/notification_worker.go b/internal/situation/notification_worker.go index ce644bd..1eb6471 100644 --- a/internal/situation/notification_worker.go +++ b/internal/situation/notification_worker.go @@ -848,12 +848,19 @@ func (w *NotificationWorker) acknowledgeDelivered(ctx context.Context, claim Not case errors.Is(err, ErrNotificationIntentSuperseded): // R4: a newer root projection replaced this one mid-flight. The // message that just went out is the older projection's; the newer - // one edits the same root next round. Expected, not a failure. + // one edits the same root next round — the store keeps a first + // post's coordinates as the Situation's root precisely so that it + // edits rather than posts again. Expected, not a failure. w.count(func(s *NotificationWorkerStats) { s.Superseded++ }) span.SetAttributes(AttrResultClass.String(DeliverResultSuperseded)) - w.auditAppend(ctx, auditKindNotificationSuperseded, intentAuditPayload(claim)) + payload := intentAuditPayload(claim) + payload["delivered_as"] = delivery.DeliveredAs + payload["channel"] = delivery.Channel + payload["message_ts"] = delivery.MessageTS + w.auditAppend(ctx, auditKindNotificationSuperseded, payload) w.logger.Info("situation: notification worker: root projection superseded mid-delivery", - "intent_id", claim.Intent.ID, "situation_id", derefString(claim.Intent.SituationID)) + "intent_id", claim.Intent.ID, "situation_id", derefString(claim.Intent.SituationID), + "channel", delivery.Channel, "message_ts", delivery.MessageTS) case errors.Is(err, ErrNotificationClaimLost): w.count(func(s *NotificationWorkerStats) { s.ClaimsLost++ }) span.SetAttributes(AttrResultClass.String(DeliverResultClaimLost)) @@ -959,6 +966,19 @@ func (w *NotificationWorker) heartbeatLoop(ctx context.Context, cancel context.C beatCtx, beatCancel := detachedWriteContext() err := w.store.HeartbeatNotificationClaim(beatCtx, claim, w.now().UTC(), w.cfg.Lease) //nolint:contextcheck // by design: detached from the possibly-canceled delivery context beatCancel() + if errors.Is(err, ErrNotificationIntentSuperseded) { + // A concurrent controller commit replaced this root + // projection. Nobody else can claim a superseded row, so + // the in-flight Slack call is still ours to finish and + // acknowledge: canceling it here would manufacture an + // uncertain outcome, and a first post Slack has already + // accepted must reach the store (MarkNotificationDelivered + // keeps its coordinates). Stop renewing a lease the row no + // longer carries. + w.logger.Info("situation: notification worker: root projection superseded mid-delivery; finishing the attempt", + "intent_id", claim.Intent.ID) + return + } if err != nil { w.logger.Warn("situation: notification worker: heartbeat failed; abandoning claim", "intent_id", claim.Intent.ID, "err", err) diff --git a/internal/situation/notification_worker_test.go b/internal/situation/notification_worker_test.go index 0d6ee64..7e263ae 100644 --- a/internal/situation/notification_worker_test.go +++ b/internal/situation/notification_worker_test.go @@ -9,6 +9,7 @@ import ( "log/slog" "strings" "sync" + "sync/atomic" "testing" "time" @@ -219,6 +220,9 @@ type nwDeliverer struct { probeCalls int calls []model.NotificationIntent deliver func(model.NotificationIntent) (NotificationDelivery, error) + // onCtx, when set, observes the delivery context after deliver + // returns — for tests that need to know whether it was canceled. + onCtx func(ctx context.Context) } func (d *nwDeliverer) Probe(context.Context) error { @@ -236,8 +240,14 @@ func (d *nwDeliverer) Deliver(ctx context.Context, intent model.NotificationInte if fn == nil { return NotificationDelivery{Channel: "C", MessageTS: "1.1", DeliveredAs: "root"}, nil } - _ = ctx - return fn(intent) + out, err := fn(intent) + d.mu.Lock() + observe := d.onCtx + d.mu.Unlock() + if observe != nil { + observe(ctx) + } + return out, err } func (d *nwDeliverer) clientMessageIDs() []string { @@ -742,6 +752,66 @@ func TestNotificationWorkerHeartbeatLossAbandonsTheClaim(t *testing.T) { } } +// TestNotificationWorkerHeartbeatSupersessionFinishesTheAttempt proves the +// other lost-fence case is NOT abandoned: a root projection superseded by a +// concurrent commit mid-flight has no other holder, so the in-flight Slack +// call completes and its outcome is acknowledged (the store keeps a first +// post's coordinates) instead of being canceled into an uncertain result +// (review round 1, R1-F3). +func TestNotificationWorkerHeartbeatSupersessionFinishesTheAttempt(t *testing.T) { + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + store := &nwStore{ + batches: [][]NotificationClaim{{nwClaim("intent-superseded-inflight", 1)}}, + heartbeatErr: ErrNotificationIntentSuperseded, + deliverAckErr: ErrNotificationIntentSuperseded, + } + released := make(chan struct{}) + var canceled atomic.Bool + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + <-released + return NotificationDelivery{Channel: "C", MessageTS: "1.1", DeliveredAs: "root"}, nil + }} + deliverer.onCtx = func(ctx context.Context) { + <-released + if ctx.Err() != nil { + canceled.Store(true) + } + } + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", + Heartbeat: time.Millisecond, + Rand: func() float64 { return 0.5 }, + }, func() time.Time { return now }, nwLogger()) + + done := make(chan struct{}) + go func() { + defer close(done) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Errorf("RunOnce: %v", err) + } + }() + for { + var beats int + store.snapshot(func(s *nwStore) { beats = s.heartbeats }) + if beats > 0 { + break + } + time.Sleep(time.Millisecond) + } + close(released) + <-done + + if canceled.Load() { + t.Fatal("the in-flight delivery was canceled on supersession; the attempt must finish and be acknowledged") + } + if got := w.Stats().ClaimsLost; got != 0 { + t.Fatalf("Stats().ClaimsLost = %d, want 0: supersession is not a lost lease", got) + } + if got := w.Stats().Superseded; got != 1 { + t.Fatalf("Stats().Superseded = %d, want 1: the outcome was acknowledged", got) + } +} + // TestNotificationWorkerStopRunsOneBoundedFinalPass proves R6's shutdown // shape: the loop ends, exactly one more pass runs under the shutdown // context, and no goroutine is left behind. diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go index 9d48c1a..ac97137 100644 --- a/internal/store/situation_history.go +++ b/internal/store/situation_history.go @@ -514,17 +514,19 @@ func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID st // FOR THE NOTIFICATION WORKER: superseding CLEARS the intent's // claim_owner/lease_expires_at (migration 0018's // `claim_owner IS NULL OR status = 'pending'` CHECK forbids leaving them on -// a non-pending row) and its retry_at. A worker holding a live claim on a -// root_sync can therefore have that claim taken out from under it by a -// concurrent controller commit, and must re-read the intent's status before -// writing any delivery outcome: a superseded row can never become -// 'delivered', because 0018's +// a non-pending row) and its retry_at, but KEEPS claim_token. A worker +// holding a live claim on a root_sync can therefore have that claim taken +// out from under it by a concurrent controller commit; a superseded row can +// never become 'delivered', because 0018's // `CHECK ((status = 'superseded') = (supersession_reason IS NOT NULL))` // aborts that write. Treat the lost claim as the expected R4 outcome — the // newer root projection supersedes what this one would have posted — not as -// a delivery failure. Supersession is performed here, inside the -// authoritative commit, precisely because the pending-root index makes it -// unorderable anywhere else; the worker must not reimplement it. +// a delivery failure. The kept token is what lets MarkNotificationDelivered +// still honor a FIRST POST Slack already accepted under it: those +// coordinates become the Situation's root, so the replacement edits them +// instead of posting a second root. Supersession is performed here, inside +// the authoritative commit, precisely because the pending-root index makes +// it unorderable anywhere else; the worker must not reimplement it. func supersedePendingRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, replacementID string) error { var pending int if err := tx.QueryRowContext(ctx, ` diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go index 785d1c7..1328bc5 100644 --- a/internal/store/situation_notifications.go +++ b/internal/store/situation_notifications.go @@ -401,6 +401,18 @@ func classifyLostNotificationClaim(ctx context.Context, q rowQueryer, intentID s // a root projection only, the Situation's durable root coordinates — the // single place slack_channel/slack_root_ts is ever written, and only when // the matching fenced root delivery is acknowledged. +// +// One acknowledgement is honored even though its row is no longer pending: +// a root projection that a concurrent controller commit SUPERSEDED while +// this very claim was in flight (supersedePendingRootSyncTx clears the +// claim but keeps its token). Slack has already accepted that post; if the +// Situation has no root yet, those coordinates ARE its root, and the +// replacement projection must edit them, not post a second root. The row +// itself stays superseded (migration 0018 forbids a superseded row becoming +// delivered) and the call still reports ErrNotificationIntentSuperseded; the +// claim token proves the acknowledging worker was the last holder, so a +// stale worker whose lease had already been reclaimed can never write +// coordinates (ADR-0049's accepted external duplicate stays external). func (s *Store) MarkNotificationDelivered(ctx context.Context, claim situation.NotificationClaim, delivery situation.NotificationDelivery, now time.Time) error { if err := validateNotificationClaim(claim); err != nil { @@ -438,7 +450,7 @@ func (s *Store) MarkNotificationDelivered(ctx context.Context, claim situation.N return fmt.Errorf("store: count delivered notification intent: %w", err) } if n != 1 { - return classifyLostNotificationClaim(ctx, tx, claim.Intent.ID) + return s.acknowledgeSupersededDeliveryTx(ctx, tx, claim, delivery) } // Which Situation's root this is comes from the intent ROW, never from @@ -458,6 +470,45 @@ func (s *Store) MarkNotificationDelivered(ctx context.Context, claim situation.N return nil } +// acknowledgeSupersededDeliveryTx handles the one lost-fence case a +// successful delivery may still act on (see MarkNotificationDelivered): the +// row was superseded under this exact claim token. For a root projection it +// records the accepted post as the Situation's root coordinates when none +// exist yet — never overwriting a root that already exists, since a +// superseded EDIT changed nothing about where the root is. Every other +// lost fence is classified as before. +func (s *Store) acknowledgeSupersededDeliveryTx(ctx context.Context, tx *sql.Tx, claim situation.NotificationClaim, + delivery situation.NotificationDelivery) error { + var status, effectClass string + var token int64 + var situationID sql.NullString + err := tx.QueryRowContext(ctx, + `SELECT status, effect_class, claim_token, situation_id FROM notification_intents WHERE id = ?`, claim.Intent.ID). + Scan(&status, &effectClass, &token, &situationID) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("store: notification intent %s: %w", claim.Intent.ID, ErrNotFound) + } + if err != nil { + return fmt.Errorf("store: classify lost notification claim: %w", err) + } + if situationmodel.IntentStatus(status) != situationmodel.IntentSuperseded { + return ErrNotificationClaimLost + } + if token != claim.ClaimToken || situationmodel.EffectClass(effectClass) != situationmodel.EffectRootSync || !situationID.Valid { + return ErrNotificationIntentSuperseded + } + if _, err := tx.ExecContext(ctx, ` + UPDATE situations SET slack_channel = ?, slack_root_ts = ? + WHERE id = ? AND slack_root_ts IS NULL`, + delivery.Channel, delivery.MessageTS, situationID.String); err != nil { + return fmt.Errorf("store: persist superseded first-post root coordinates: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("store: commit superseded first-post root coordinates: %w", err) + } + return ErrNotificationIntentSuperseded +} + // RetryNotificationIntent releases a claimed intent back for a later retry. // It keeps the intent pending and keeps its attempt count: there is no // attempt ceiling anywhere in this lifecycle. diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index b8cf6c3..e701698 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -1011,3 +1011,100 @@ func TestCommitWithheldRootRetiresThePendingRootItReplaces(t *testing.T) { t.Fatalf("%d root projection(s) still pending behind a withheld replacement", pending) } } + +// ---------------------------------------------------------------------- +// In-flight supersession keeps a successful first post (review round 1, +// R1-F3). +// ---------------------------------------------------------------------- + +func TestMarkNotificationDeliveredSupersededFirstPostKeepsCoordinates(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + id, first := snSeedOneCycle(t, st, "review-inflight-post", now) + posted := snClaimOne(t, st, now) + if posted.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("claimed %s, want the first root projection", posted.Intent.EffectClass) + } + // Slack accepts the first post, but another controller commit replaces + // its projection before the success response is acknowledged. + snCommit(t, st, id, &first, now.Add(time.Second)) + err := st.MarkNotificationDelivered(ctx, posted, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: "100.1", DeliveredAs: "root"}, now.Add(2*time.Second)) + if !errors.Is(err, ErrNotificationIntentSuperseded) { + t.Fatalf("ack error = %v, want ErrNotificationIntentSuperseded: the row itself stays superseded", err) + } + _, ts, ok, readErr := st.GetSituationRootCoordinates(ctx, id) + if readErr != nil { + t.Fatal(readErr) + } + if !ok || ts != "100.1" { + t.Fatalf("successful first post lost coordinates after supersession: published=%v ts=%q; the replacement would post a second root", ok, ts) + } + if got := snIntent(t, st, posted.Intent.ID).Status; got != situationmodel.IntentSuperseded { + t.Fatalf("superseded row status = %s, want superseded (never delivered)", got) + } + // The replacement projection now edits that root: it is claimable and + // its delivery lands on the same coordinates. + replacement := snClaimOne(t, st, now.Add(3*time.Second)) + if replacement.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("claimed %s, want the replacement root projection", replacement.Intent.EffectClass) + } + snDeliver(t, st, replacement, "100.1", now.Add(4*time.Second)) + _, ts, _, _ = st.GetSituationRootCoordinates(ctx, id) + if ts != "100.1" { + t.Fatalf("root coordinates = %q after the replacement delivered, want the first post's 100.1", ts) + } +} + +func TestMarkNotificationDeliveredSupersededAckFromAStaleHolderWritesNothing(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + id, first := snSeedOneCycle(t, st, "review-stale-holder", now) + stale := snClaimOne(t, st, now) + // The stale holder's five-minute lease expires and a second worker + // reclaims the same projection (a new claim token); only then is it + // superseded. + if _, err := st.RecoverExpiredNotificationClaims(ctx, now.Add(6*time.Minute)); err != nil { + t.Fatal(err) + } + fresh := snClaimOne(t, st, now.Add(6*time.Minute)) + if fresh.ClaimToken == stale.ClaimToken { + t.Fatal("fixture: the reclaim did not advance the claim token") + } + snCommit(t, st, id, &first, now.Add(7*time.Minute)) + err := st.MarkNotificationDelivered(ctx, stale, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: "77.7", DeliveredAs: "root"}, now.Add(8*time.Minute)) + if !errors.Is(err, ErrNotificationIntentSuperseded) { + t.Fatalf("ack error = %v, want ErrNotificationIntentSuperseded", err) + } + if _, _, ok, _ := st.GetSituationRootCoordinates(ctx, id); ok { + t.Fatal("a stale holder's acknowledgement wrote root coordinates; only the last claim holder may") + } +} + +func TestMarkNotificationDeliveredSupersededEditNeverMovesTheRoot(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + id, first := snSeedOneCycle(t, st, "review-superseded-edit", now) + snDeliver(t, st, snClaimOne(t, st, now), "100.1", now) + snDeliver(t, st, snClaimOne(t, st, now), "100.2", now) + second := snCommit(t, st, id, &first, now.Add(time.Minute)) + edit := snClaimOne(t, st, now.Add(time.Minute)) + if edit.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("claimed %s, want the root edit", edit.Intent.EffectClass) + } + // A third material commit (the contract hands the next move back to + // AlertINT) replaces the claimed edit's projection mid-flight. + snCommitWith(t, st, id, &second, shRunningTriageContract(now.Add(3*time.Minute)), now.Add(2*time.Minute)) + err := st.MarkNotificationDelivered(ctx, edit, + situation.NotificationDelivery{Channel: "C-sit", MessageTS: "100.1", DeliveredAs: "root"}, now.Add(3*time.Minute)) + if !errors.Is(err, ErrNotificationIntentSuperseded) { + t.Fatalf("ack error = %v, want ErrNotificationIntentSuperseded", err) + } + if _, ts, _, _ := st.GetSituationRootCoordinates(ctx, id); ts != "100.1" { + t.Fatalf("root coordinates = %q, want the original 100.1: an edit never re-anchors a root", ts) + } +} From c10aad115ff0413ba8b2664d4562e40f752a5bc4 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 13:06:32 +0300 Subject: [PATCH 23/31] fix(store): blocked and failed effects hold their Situation's queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim ranking included only pending rows. Once a handoff's root edit became blocked_configuration (or failed) it vanished from the per-Situation ordering, and because the original root coordinates still existed the handoff was claimable immediately, even though the projection it depends on never delivered. The same gap let reactivation deliver newer history before an older blocked reply (review round 1, R1-F2). Ranking now includes blocked_configuration and failed rows as non-claimable holders of the queue head: an existing timestamp proves a root exists, not that this projection reached it, and a blocked reply holds every later reply. So that an obsolete root projection is never a permanent blocker, a newer root projection now supersedes every LIVE older one at commit — pending, blocked, or failed — leaving one live root per Situation; the obligation continues in the replacement, and a later valid projection repairs a failed root with no redrive. Migration 0020 replaces 0018's "supersede from pending only" trigger accordingly (delivered, withheld, and superseded rows still never become superseded); MaxSchemaVersion is 20. Reactivation simplifies to "the one live root, if blocked, becomes pending". Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- ...0020_notification_supersede_live_roots.sql | 25 +++ ...notification_blocked_index_upgrade_test.go | 2 +- internal/store/notification_gaps.go | 45 +---- ...ation_supersede_live_roots_upgrade_test.go | 165 ++++++++++++++++ internal/store/situation_history.go | 50 +++-- internal/store/situation_notifications.go | 19 +- .../store/situation_notifications_test.go | 176 ++++++++++++++++-- internal/store/store_test.go | 2 +- 8 files changed, 405 insertions(+), 79 deletions(-) create mode 100644 internal/store/migrations/0020_notification_supersede_live_roots.sql create mode 100644 internal/store/notification_supersede_live_roots_upgrade_test.go diff --git a/internal/store/migrations/0020_notification_supersede_live_roots.sql b/internal/store/migrations/0020_notification_supersede_live_roots.sql new file mode 100644 index 0000000..b742060 --- /dev/null +++ b/internal/store/migrations/0020_notification_supersede_live_roots.sql @@ -0,0 +1,25 @@ +-- SPDX-License-Identifier: FSL-1.1-ALv2 +-- +-- One trigger replaced, no table, column, index, or row changed: a newer +-- root projection may now supersede a configuration-blocked or failed +-- older one, not only a pending one. +-- +-- Migration 0018 let only a PENDING root_sync become superseded, calling +-- blocked/failed "already resolved outcomes, not live candidates a newer +-- root_sync coalesces away". Review round 1 (R1-F2) showed why that cannot +-- hold once ordering is honest: a blocked or failed root projection still +-- names the Situation's unmet publication obligation, so it must keep the +-- head of its Situation's claim queue — a handoff whose root edit never +-- delivered is not claimable just because an older root exists. A row that +-- holds the queue but can never be retired by newer state would be a +-- permanent blocker. So the three LIVE statuses — pending, +-- blocked_configuration, failed — are all coalesced by the next root +-- projection; delivered, withheld, and superseded rows still never are, +-- and a superseded row still records why and by what. +-- +-- This migration fabricates nothing for any Situation that predates it. +-- ---------------------------------------------------------------------- +DROP TRIGGER notification_intents_supersede_from_pending_only; +CREATE TRIGGER notification_intents_supersede_from_live_only BEFORE UPDATE OF status ON notification_intents +WHEN NEW.status = 'superseded' AND OLD.status NOT IN ('pending', 'blocked_configuration', 'failed') +BEGIN SELECT RAISE(ABORT, 'only a live root_sync intent (pending, blocked_configuration, or failed) may become superseded'); END; diff --git a/internal/store/notification_blocked_index_upgrade_test.go b/internal/store/notification_blocked_index_upgrade_test.go index 5d91aab..5cd74be 100644 --- a/internal/store/notification_blocked_index_upgrade_test.go +++ b/internal/store/notification_blocked_index_upgrade_test.go @@ -130,7 +130,7 @@ func TestNotificationBlockedIndexUpgrade_AddsThePartialIndexAndFabricatesNothing if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - if got != 19 { + if got != 20 { t.Fatalf("MaxSchemaVersion = %d, want 19", got) } diff --git a/internal/store/notification_gaps.go b/internal/store/notification_gaps.go index 77adf0f..cf9b9e6 100644 --- a/internal/store/notification_gaps.go +++ b/internal/store/notification_gaps.go @@ -506,29 +506,13 @@ type liveRootProjection struct { // reactivateBlockedRootSyncTx restores exactly one pending root projection // for situationID and reports how many blocked roots it reactivated (0 or 1). // -// The newest live projection is the one corrected configuration should -// deliver — it renders current state, and every older one would render state -// already superseded by it. Two cases: -// -// - The newest is ALREADY pending. It holds the slot and says everything -// the older blocked ones would; they stay blocked_configuration, which -// migration 0018 explicitly calls a resolved outcome ("never one already -// delivered/blocked/failed/withheld ... not live candidates a newer -// root_sync coalesces away"). Nothing is stranded: the pending projection -// delivers the root coordinates every dependent effect waits on. -// -// - The newest is blocked. It is reactivated, and every older live -// projection is coalesced into it through Task 5's own -// supersedePendingRootSyncTx — the same supersession a newer commit -// performs. An older BLOCKED one reaches `superseded` the only way the -// schema permits, by being reactivated first: that is exactly what -// happened (corrected configuration returned it to pending) immediately -// followed by the newer projection coalescing it. -// -// The order is what keeps the unique index satisfied at every step: the -// pre-existing pending row is retired first, then each older blocked row is -// made pending and immediately coalesced, and only then does the keeper -// become pending. At no point do two root projections hold the slot. +// A newer root projection supersedes every older live one at commit +// (supersedeLiveRootSyncTx), so a Situation normally holds ONE live root: +// if it is pending, corrected configuration has nothing to do here — that +// projection delivers the coordinates every dependent effect waits on; if +// it is blocked, it becomes pending. Any older live projection that somehow +// survived is coalesced into the newest one first, so migration 0018's +// single-pending-root index is satisfied at every step. func reactivateBlockedRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, nowStr string) (int, error) { rows, err := tx.QueryContext(ctx, ` SELECT id, status FROM notification_intents @@ -549,22 +533,9 @@ func reactivateBlockedRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, n if keeper.status == string(situationmodel.IntentPending) { return 0, nil } - - // Retire whichever projection currently holds the pending slot, if any. - if err := supersedePendingRootSyncTx(ctx, tx, situationID, keeper.id); err != nil { + if err := supersedeLiveRootSyncTx(ctx, tx, situationID, keeper.id); err != nil { return 0, err } - for _, older := range live[:len(live)-1] { - if older.status != string(situationmodel.IntentBlockedConfiguration) { - continue // already retired by the supersession above - } - if err := setNotificationIntentPendingTx(ctx, tx, older.id, "blocked_configuration", nowStr); err != nil { - return 0, err - } - if err := supersedePendingRootSyncTx(ctx, tx, situationID, keeper.id); err != nil { - return 0, err - } - } if err := setNotificationIntentPendingTx(ctx, tx, keeper.id, "blocked_configuration", nowStr); err != nil { return 0, err } diff --git a/internal/store/notification_supersede_live_roots_upgrade_test.go b/internal/store/notification_supersede_live_roots_upgrade_test.go new file mode 100644 index 0000000..936c824 --- /dev/null +++ b/internal/store/notification_supersede_live_roots_upgrade_test.go @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package store + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + "time" +) + +// ---------------------------------------------------------------------- +// Migration 0020 upgrade test: a populated migration-19 database must have +// its supersession trigger replaced (a blocked or failed root projection +// may now be superseded by a newer one; a delivered one still may not), +// keep MaxSchemaVersion honest at 20, pass PRAGMA foreign_key_check, and +// fabricate no rows. +// ---------------------------------------------------------------------- + +func seedMigration19SupersedeFixture(t *testing.T, path string) (blockedRootID, deliveredRootID string) { + t.Helper() + ctx := context.Background() + + db, err := sql.Open("sqlite", buildDSN(path)) + if err != nil { + t.Fatalf("open fixture db: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ) STRICT; + `); err != nil { + t.Fatalf("create schema_migrations: %v", err) + } + migrations, err := loadMigrations() + if err != nil { + t.Fatalf("load migrations: %v", err) + } + fixture := &Store{db: db} + for _, m := range migrations { + if m.version > 19 { + continue + } + if err := fixture.applyMigration(ctx, m); err != nil { + t.Fatalf("apply migration %d: %v", m.version, err) + } + } + + now := time.Now().UTC() + situationID := "sit-supersede-live" + insertOperationalIncident(ctx, t, fixture, "inc-supersede-live", "group-supersede-live") + if err := insertSituation(ctx, fixture, situationRow{ + id: situationID, groupKey: "group-supersede-live", lifecycle: "active", + }); err != nil { + t.Fatalf("insert situation: %v", err) + } + transitionID := "tr-supersede-live" + if _, err := db.ExecContext(ctx, ` + INSERT INTO situation_transitions ( + id, situation_id, sequence, input_version, material_fact_hash, lifecycle, attention, + action_contract_json, reason, journal_kind, journal_json, projection_json, + evidence_refs_json, actor, created_at + ) VALUES (?, ?, 1, 1, 'sha256:supersede-live', 'active', 'observe', + '{}', 'first_authoritative_state', 'publication', '{}', '{}', '[]', + 'deterministic_controller', ?)`, + transitionID, situationID, canonicalTime(now)); err != nil { + t.Fatalf("insert transition: %v", err) + } + blockedRootID = "root-blocked" + deliveredRootID = "root-delivered" + for _, row := range []struct{ id, status, extra string }{ + {blockedRootID, "blocked_configuration", "last_error_class"}, + {deliveredRootID, "delivered", ""}, + } { + if _, err := db.ExecContext(ctx, ` + INSERT INTO notification_intents ( + id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, summary_version, requires_root, + main_channel_poke, client_message_id, status, last_error_class, + delivered_as, channel, message_ts, delivered_at, created_at + ) VALUES (?, ?, 'root_sync', ?, ?, 1, 1, 0, 0, ?, ?, ?, ?, ?, ?, ?, ?)`, + row.id, "idem:"+row.id, situationID, transitionID, "client:"+row.id, row.status, + nullIf(row.extra == "", "channel_not_found"), + nullIf(row.status != "delivered", "root"), nullIf(row.status != "delivered", "C"), + nullIf(row.status != "delivered", "1.1"), nullIf(row.status != "delivered", canonicalTime(now)), + canonicalTime(now)); err != nil { + t.Fatalf("seed %s root: %v", row.status, err) + } + } + return blockedRootID, deliveredRootID +} + +// nullIf returns NULL when cond holds, else value. +func nullIf(cond bool, value string) any { + if cond { + return nil + } + return value +} + +func TestNotificationSupersedeLiveRootsUpgrade(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "upgrade-20.db") + blockedRootID, deliveredRootID := seedMigration19SupersedeFixture(t, path) + + st, err := Open(ctx, path) + if err != nil { + t.Fatalf("open (apply migration 0020): %v", err) + } + defer func() { _ = st.Close() }() + + got, err := MaxSchemaVersion() + if err != nil { + t.Fatalf("MaxSchemaVersion: %v", err) + } + if got != 20 { + t.Fatalf("MaxSchemaVersion = %d, want 20", got) + } + var version int + if err := st.db.QueryRowContext(ctx, `SELECT MAX(version) FROM schema_migrations`).Scan(&version); err != nil || version != 20 { + t.Fatalf("applied schema version = %d (err=%v), want 20", version, err) + } + var fkViolations int + if err := st.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pragma_foreign_key_check`).Scan(&fkViolations); err != nil || fkViolations != 0 { + t.Fatalf("foreign_key_check violations = %d (err=%v), want 0", fkViolations, err) + } + var triggers string + rows, err := st.db.QueryContext(ctx, `SELECT name FROM sqlite_master WHERE type = 'trigger' AND name LIKE 'notification_intents_supersede%'`) + if err != nil { + t.Fatalf("list triggers: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan trigger: %v", err) + } + triggers += name + ";" + } + _ = rows.Close() + if strings.Contains(triggers, "from_pending_only") || !strings.Contains(triggers, "from_live_only") { + t.Fatalf("supersession triggers after upgrade = %q, want only notification_intents_supersede_from_live_only", triggers) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents`); n != 2 { + t.Fatalf("notification intents after upgrade = %d, want the 2 seeded rows (nothing fabricated)", n) + } + + // A blocked root projection may now be superseded by a newer one. + if _, err := st.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'superseded', supersession_reason = 'newer_root_projection', + replacement_intent_id = ?, last_error_class = NULL + WHERE id = ?`, deliveredRootID, blockedRootID); err != nil { + t.Fatalf("supersede a blocked root after 0020: %v", err) + } + // A delivered one still may not. + if _, err := st.db.ExecContext(ctx, ` + UPDATE notification_intents SET status = 'superseded', supersession_reason = 'newer_root_projection', + replacement_intent_id = ?, delivered_as = NULL, channel = NULL, message_ts = NULL, delivered_at = NULL + WHERE id = ?`, blockedRootID, deliveredRootID); err == nil || !strings.Contains(err.Error(), "live root_sync") { + t.Fatalf("superseding a delivered root = %v, want the 0020 trigger's rejection", err) + } +} diff --git a/internal/store/situation_history.go b/internal/store/situation_history.go index ac97137..8c165ec 100644 --- a/internal/store/situation_history.go +++ b/internal/store/situation_history.go @@ -490,7 +490,7 @@ func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID st if intent.EffectClass != situationmodel.EffectRootSync { continue } - if err := supersedePendingRootSyncTx(ctx, tx, situationID, intent.ID); err != nil { + if err := supersedeLiveRootSyncTx(ctx, tx, situationID, intent.ID); err != nil { return err } break // PlanNotificationIntents never plans two root projections per commit. @@ -503,13 +503,28 @@ func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID st return nil } -// supersedePendingRootSyncTx retires every currently-pending root_sync for -// situationID in favour of replacementID. replacementID is inserted later -// in this same transaction, so foreign-key enforcement is deferred to -// COMMIT for the duration: the self-referencing replacement_intent_id FK -// and the "at most one pending root_sync" index would otherwise make the -// two writes impossible to order. Deferral changes when a violation is -// reported, never whether the transaction is atomic. +// supersedeLiveRootSyncTx retires every LIVE root_sync for situationID — +// pending, configuration-blocked, or failed — in favour of replacementID +// (itself excluded, so a caller may name a row that already exists). +// spec.md: "Older root projections for the same Situation may become +// superseded by the latest root sync." Retiring the blocked and failed ones +// too, not only the pending one, is what keeps an obsolete projection from +// holding its Situation's queue forever: a blocked or failed root still +// ranks at the head of the claim order (ClaimNotificationIntents, review +// round 1 R1-F2), and a newer projection renders everything it would have, +// so the delivery obligation continues in the replacement rather than +// waiting on a reactivation or redrive of state nobody wants on screen. +// The invariant this leaves is one live root projection per Situation. +// Migration 0020 owns the trigger that permits exactly these three source +// statuses; a delivered, withheld, or already-superseded row still never +// becomes superseded. +// +// replacementID may be inserted later in this same transaction, so +// foreign-key enforcement is deferred to COMMIT for the duration: the +// self-referencing replacement_intent_id FK and the "at most one pending +// root_sync" index would otherwise make the two writes impossible to +// order. Deferral changes when a violation is reported, never whether the +// transaction is atomic. // // FOR THE NOTIFICATION WORKER: superseding CLEARS the intent's // claim_owner/lease_expires_at (migration 0018's @@ -527,14 +542,16 @@ func insertNotificationIntentsTx(ctx context.Context, tx *sql.Tx, situationID st // instead of posting a second root. Supersession is performed here, inside // the authoritative commit, precisely because the pending-root index makes // it unorderable anywhere else; the worker must not reimplement it. -func supersedePendingRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, replacementID string) error { - var pending int +func supersedeLiveRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, replacementID string) error { + var live int if err := tx.QueryRowContext(ctx, ` SELECT COUNT(*) FROM notification_intents - WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, situationID).Scan(&pending); err != nil { - return fmt.Errorf("store: count pending root projections: %w", err) + WHERE situation_id = ? AND effect_class = 'root_sync' + AND status IN ('pending', 'blocked_configuration', 'failed') AND id <> ?`, + situationID, replacementID).Scan(&live); err != nil { + return fmt.Errorf("store: count live root projections: %w", err) } - if pending == 0 { + if live == 0 { return nil } if _, err := tx.ExecContext(ctx, `PRAGMA defer_foreign_keys = ON`); err != nil { @@ -544,9 +561,10 @@ func supersedePendingRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, re UPDATE notification_intents SET status = 'superseded', supersession_reason = ?, replacement_intent_id = ?, claim_owner = NULL, lease_expires_at = NULL, retry_at = NULL - WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, - SupersessionReasonNewerRootProjection, replacementID, situationID); err != nil { - return fmt.Errorf("store: supersede pending root projections: %w", err) + WHERE situation_id = ? AND effect_class = 'root_sync' + AND status IN ('pending', 'blocked_configuration', 'failed') AND id <> ?`, + SupersessionReasonNewerRootProjection, replacementID, situationID, replacementID); err != nil { + return fmt.Errorf("store: supersede live root projections: %w", err) } return nil } diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go index 1328bc5..eb1ec3d 100644 --- a/internal/store/situation_notifications.go +++ b/internal/store/situation_notifications.go @@ -124,6 +124,16 @@ func validateNotificationErrorClass(class string) error { // Transition SEQUENCE (effect class decides only ties within one // sequence), so a later immutable entry can never pass an // earlier pending one — including one merely waiting out a retry delay. +// A blocked_configuration or failed effect still holds the head of +// that queue without being claimable itself: a handoff whose root edit +// never delivered is not claimable just because an OLDER root exists +// (an existing timestamp proves a root exists, not that this +// projection reached it), and a blocked reply holds every later reply +// so reactivation can never deliver newer history before older +// (review round 1, R1-F2). Only a live root projection can hold the +// queue: a newer root projection supersedes an older pending, blocked, +// OR failed one at commit (supersedeLiveRootSyncTx), so an obsolete +// root is never a permanent blocker. // // A gap generation gates the whole claim: while one is open nothing is // claimable at all (Slack is down and the recovery notice must precede the @@ -271,6 +281,7 @@ func dueNotificationIntentIDsTx(ctx context.Context, tx *sql.Tx, gate deliveryGa ni.transition_sequence AS transition_sequence, `+notificationRootFirst+` AS root_first, `+notificationClassRank+` AS class_rank, + (ni.status = 'pending') AS claimable, (ni.claim_owner IS NULL OR ni.lease_expires_at <= ?) AS unleased, (ni.retry_at IS NULL OR ni.retry_at <= ?) AS due, (ni.requires_root = 0 OR (s.slack_channel IS NOT NULL AND s.slack_root_ts IS NOT NULL)) AS root_ready, @@ -280,10 +291,10 @@ func dueNotificationIntentIDsTx(ctx context.Context, tx *sql.Tx, gate deliveryGa ) AS rn FROM notification_intents ni LEFT JOIN situations s ON s.id = ni.situation_id - WHERE ni.status = 'pending' + WHERE ni.status IN ('pending', 'blocked_configuration', 'failed') ) SELECT id FROM ranked - WHERE (situation_id IS NULL OR rn = 1) AND unleased AND due AND root_ready + WHERE (situation_id IS NULL OR rn = 1) AND claimable AND unleased AND due AND root_ready `+notificationClaimOrder+` LIMIT ?`, nowStr, nowStr, limit) if err != nil { @@ -404,7 +415,7 @@ func classifyLostNotificationClaim(ctx context.Context, q rowQueryer, intentID s // // One acknowledgement is honored even though its row is no longer pending: // a root projection that a concurrent controller commit SUPERSEDED while -// this very claim was in flight (supersedePendingRootSyncTx clears the +// this very claim was in flight (supersedeLiveRootSyncTx clears the // claim but keeps its token). Slack has already accepted that post; if the // Situation has no root yet, those coordinates ARE its root, and the // replacement projection must edit them, not post a second root. The row @@ -671,7 +682,7 @@ func clearPendingRootForRedriveTx(ctx context.Context, tx *sql.Tx, situationID, if pendingCreatedAt > createdAt || (pendingCreatedAt == createdAt && pendingID > intentID) { return ErrNewerRootProjectionPending } - return supersedePendingRootSyncTx(ctx, tx, situationID, intentID) + return supersedeLiveRootSyncTx(ctx, tx, situationID, intentID) } // GetSituationRootCoordinates reads situationID's durable Slack root diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index e701698..9f9895a 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -5,6 +5,8 @@ package store import ( "context" "errors" + "fmt" + "strings" "testing" "time" @@ -601,20 +603,23 @@ func TestNotificationSupersessionTakesTheClaimFromAnInFlightRootSync(t *testing. } after := snIntent(t, st, claim.Intent.ID) if after.Status != situationmodel.IntentSuperseded || after.DeliveredAt != nil { - t.Fatalf("superseded root after a refused ack = %+v, want it untouched", after) + t.Fatalf("superseded root after its ack = %+v, want the row itself untouched", after) } - if _, _, ok, err := st.GetSituationRootCoordinates(ctx, sitID); err != nil || ok { - t.Fatalf("root coordinates after a refused ack = (ok=%v, err=%v), want (false, nil)", ok, err) + // The post Slack accepted IS the Situation's root: its coordinates are + // kept so the replacement edits it rather than posting a second one + // (review round 1, R1-F3). + if _, ts, ok, err := st.GetSituationRootCoordinates(ctx, sitID); err != nil || !ok || ts != "100.1" { + t.Fatalf("root coordinates after the superseded first post's ack = (ok=%v, ts=%q, err=%v), want (true, 100.1, nil)", ok, ts, err) } // The replacement projection is the claimable head, and its own - // delivery still works normally. + // delivery edits that same root. next := snClaimOne(t, st, now.Add(3*time.Minute)) replacement := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) if next.Intent.ID != replacement.ID { t.Fatalf("next head = %s, want the replacement root %s", next.Intent.ID, replacement.ID) } - snDeliver(t, st, next, "101.1", now.Add(3*time.Minute)) + snDeliver(t, st, next, "100.1", now.Add(3*time.Minute)) } // TestNotificationSupersessionLeavesImmutableEntriesAndOtherEpisodes proves @@ -827,10 +832,16 @@ func TestNotificationAckReactivationKeepsOnePendingRootProjection(t *testing.T) if err := st.BlockNotificationConfiguration(ctx, blocked, "channel_not_found", now); err != nil { t.Fatalf("BlockNotificationConfiguration: %v", err) } - // A later material commit inserts a NEW root projection. Supersession - // only ever retires a pending one, so the blocked one survives beside it. + // A later material commit inserts a NEW root projection and supersedes + // the blocked one at commit: an obsolete root is never left holding + // its Situation's queue. second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) secondRoot := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) + if older := snIntent(t, st, firstRoot.ID); older.Status != situationmodel.IntentSuperseded || + older.ReplacementIntentID == nil || *older.ReplacementIntentID != secondRoot.ID { + t.Fatalf("blocked root after a newer commit = %q (replacement %v), want superseded by %s", + older.Status, older.ReplacementIntentID, secondRoot.ID) + } n, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(2*time.Minute)) if err != nil { @@ -843,7 +854,7 @@ func TestNotificationAckReactivationKeepsOnePendingRootProjection(t *testing.T) t.Fatalf("newer root status = %q, want pending", got.Status) } if got := snIntent(t, st, firstRoot.ID); got.Status == situationmodel.IntentPending { - t.Fatal("the superseded-by-newer blocked root must not be returned to pending") + t.Fatal("the superseded blocked root must not be returned to pending") } if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, @@ -914,11 +925,15 @@ func TestNotificationAckReactivationCoalescesOlderBlockedRootProjections(t *test } } -// TestNotificationAckRedriveRefusesBehindANewerRootProjection proves the -// same uniqueness hazard cannot reach an operator redrive either: redriving -// an older failed root behind a newer pending one is refused with a typed -// error, not a raw constraint violation. -func TestNotificationAckRedriveRefusesBehindANewerRootProjection(t *testing.T) { +// TestNotificationAckFailedRootIsSupersededByANewerProjection proves a +// failed root projection is not a permanent blocker: the next commit's +// projection supersedes it at commit and is itself claimable at the head +// of the queue (a failed row holds the queue until then — see +// TestNotificationClaimBlockedRootEditHoldsItsHandoff), so a later valid +// projection repairs the Situation with no operator redrive. The +// superseded row is then no longer failed and cannot be redriven — the +// obligation lives in its replacement. +func TestNotificationAckFailedRootIsSupersededByANewerProjection(t *testing.T) { st := newTestStore(t) ctx := context.Background() now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) @@ -929,18 +944,25 @@ func TestNotificationAckRedriveRefusesBehindANewerRootProjection(t *testing.T) { if err := st.FailNotificationIntent(ctx, failed, "invalid_payload", now); err != nil { t.Fatalf("FailNotificationIntent: %v", err) } + // The failed root holds the queue: its dependent journal entry is not + // claimable behind it. + if claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(30*time.Second), time.Minute, 25); err != nil || len(claims) != 0 { + t.Fatalf("claimed %v behind a failed root (err=%v), want nothing", snClasses(claims), err) + } second := snCommit(t, st, sitID, &first, now.Add(time.Minute)) secondRoot := shIntentOfClass(t, second.History.Intents, situationmodel.EffectRootSync) - err := st.RedriveFailedNotificationIntent(ctx, firstRoot.ID, now.Add(2*time.Minute)) - if !errors.Is(err, ErrNewerRootProjectionPending) { - t.Fatalf("redrive behind a newer pending root = %v, want ErrNewerRootProjectionPending", err) + older := snIntent(t, st, firstRoot.ID) + if older.Status != situationmodel.IntentSuperseded || older.ReplacementIntentID == nil || *older.ReplacementIntentID != secondRoot.ID { + t.Fatalf("failed root after a newer commit = %q (replacement %v), want superseded by %s", + older.Status, older.ReplacementIntentID, secondRoot.ID) } - if got := snIntent(t, st, firstRoot.ID); got.Status != situationmodel.IntentFailed { - t.Fatalf("refused redrive changed the failed root to %q", got.Status) + if err := st.RedriveFailedNotificationIntent(ctx, firstRoot.ID, now.Add(2*time.Minute)); !errors.Is(err, ErrNotFound) { + t.Fatalf("redrive of a superseded root = %v, want ErrNotFound: it is no longer failed", err) } - if got := snIntent(t, st, secondRoot.ID); got.Status != situationmodel.IntentPending { - t.Fatalf("newer root status = %q, want an untouched pending", got.Status) + claim := snClaimOne(t, st, now.Add(2*time.Minute)) + if claim.Intent.ID != secondRoot.ID { + t.Fatalf("claimed %s, want the replacement root %s at the head of the queue", claim.Intent.ID, secondRoot.ID) } } @@ -1108,3 +1130,117 @@ func TestMarkNotificationDeliveredSupersededEditNeverMovesTheRoot(t *testing.T) t.Fatalf("root coordinates = %q, want the original 100.1: an edit never re-anchors a root", ts) } } + +// ---------------------------------------------------------------------- +// Blocked and failed effects hold the queue (review round 1, R1-F2). +// ---------------------------------------------------------------------- + +func TestNotificationClaimBlockedRootEditHoldsItsHandoff(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + id, first := snSeedOneCycle(t, st, "review-blocked-edit", now) + snDeliver(t, st, snClaimOne(t, st, now), "100.1", now) + snDeliver(t, st, snClaimOne(t, st, now), "100.2", now) + now = now.Add(time.Minute) + snCommit(t, st, id, &first, now) + root := snClaimOne(t, st, now) + if root.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatal("expected root edit") + } + if err := st.BlockNotificationConfiguration(ctx, root, "missing_scope", now); err != nil { + t.Fatal(err) + } + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now, time.Minute, 25) + if err != nil { + t.Fatal(err) + } + if len(claims) != 0 { + t.Fatalf("claimed %v before its blocked root edit was delivered: an existing timestamp proves a root exists, not that this projection reached it", snClasses(claims)) + } + // Corrected configuration returns the edit to pending; it delivers + // FIRST, and only then is the handoff claimable. + if _, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + edit := snClaimOne(t, st, now.Add(time.Minute)) + if edit.Intent.ID != root.Intent.ID { + t.Fatalf("claimed %s after reactivation, want the reactivated root edit %s", edit.Intent.ID, root.Intent.ID) + } + snDeliver(t, st, edit, "100.1", now.Add(time.Minute)) + handoff := snClaimOne(t, st, now.Add(2*time.Minute)) + if handoff.Intent.EffectClass != situationmodel.EffectBroadcastHandoff { + t.Fatalf("claimed %s after the root edit delivered, want the handoff", handoff.Intent.EffectClass) + } +} + +func TestNotificationClaimBlockedReplyHoldsLaterHistoryInOrder(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + id, first := snSeedOneCycle(t, st, "review-blocked-reply", now) + snDeliver(t, st, snClaimOne(t, st, now), "100.1", now) // root + reply1 := snClaimOne(t, st, now) // journal entry #1 + if reply1.Intent.EffectClass != situationmodel.EffectThreadAppend { + t.Fatalf("claimed %s, want the first journal entry", reply1.Intent.EffectClass) + } + if err := st.BlockNotificationConfiguration(ctx, reply1, "channel_not_found", now); err != nil { + t.Fatal(err) + } + // A later commit (an operator handoff) adds a root edit and a newer + // journal effect. The root edit is claimable (root projections come + // first); the newer effect is not, because the blocked older entry + // holds the queue. + now = now.Add(time.Minute) + snCommit(t, st, id, &first, now) + edit := snClaimOne(t, st, now) + if edit.Intent.EffectClass != situationmodel.EffectRootSync { + t.Fatalf("claimed %s, want the root edit", edit.Intent.EffectClass) + } + snDeliver(t, st, edit, "100.1", now) + if claims, err := st.ClaimNotificationIntents(ctx, snOwner, now, time.Minute, 25); err != nil || len(claims) != 0 { + t.Fatalf("claimed %v behind a blocked older journal entry (err=%v); reactivation must never deliver newer history before older", snClasses(claims), err) + } + if _, err := st.ReactivateConfigurationBlocked(ctx, 1, now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now.Add(time.Minute), 5*time.Minute, 25) + if err != nil || len(claims) != 1 { + t.Fatalf("claimed %d after reactivation (err=%v), want the older entry:%s", len(claims), err, snDump(t, st)) + } + older := claims[0] + if older.Intent.ID != reply1.Intent.ID { + t.Fatalf("claimed %s after reactivation, want the older entry %s first", older.Intent.ID, reply1.Intent.ID) + } + snDeliver(t, st, older, "100.3", now.Add(time.Minute)) + newer := snClaimOne(t, st, now.Add(2*time.Minute)) + if newer.Intent.EffectClass == situationmodel.EffectRootSync || *newer.Intent.TransitionSequence <= *reply1.Intent.TransitionSequence { + t.Fatalf("claimed %s #%d after the older entry delivered, want the newer journal effect", + newer.Intent.EffectClass, *newer.Intent.TransitionSequence) + } +} + +// snDump renders every notification intent as one line per row, for +// failure messages. +func snDump(t *testing.T, st *Store) string { + t.Helper() + rows, err := st.db.QueryContext(context.Background(), ` + SELECT id, effect_class, status, COALESCE(transition_sequence, 0), COALESCE(retry_at, ''), COALESCE(claim_owner, ''), + COALESCE(last_error_class, ''), requires_root + FROM notification_intents ORDER BY created_at, id`) + if err != nil { + t.Fatalf("dump intents: %v", err) + } + defer func() { _ = rows.Close() }() + var out strings.Builder + for rows.Next() { + var id, class, status, retryAt, owner, errClass string + var seq, requiresRoot int + if err := rows.Scan(&id, &class, &status, &seq, &retryAt, &owner, &errClass, &requiresRoot); err != nil { + t.Fatalf("scan intent: %v", err) + } + fmt.Fprintf(&out, "\n %s class=%s status=%s seq=%d retry_at=%q owner=%q err=%q requires_root=%d", + id[:8], class, status, seq, retryAt, owner, errClass, requiresRoot) + } + return out.String() +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 03b9d5f..8cc17ae 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -451,7 +451,7 @@ func TestMaxSchemaVersion(t *testing.T) { // migration ownership") plus 0019, which adds one partial index and no // schema of its own — 0017 and 0018 are final and are never edited, so a // new migration is the only sanctioned way to add it. - if got != 19 { + if got != 20 { t.Errorf("MaxSchemaVersion = %d, want 19", got) } } From 47260be58beb745587750ad6ea2f30945ac03496 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 13:17:13 +0300 Subject: [PATCH 24/31] fix(situation): a token probe is not delivery health MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Probe is auth.test, yet each successful probe cleared first_failure_at and could recover an open Delivery gap. With authentication healthy but chat.postMessage/chat.update failing, the worker reset the continuous delivery-failure window on every probe, so the mandatory five-minute gap never opened and a gap could be declared recovered while writes still failed (review round 1, R1-F4). Now only an actual delivery success clears the window. A successful probe still reactivates blocked configuration (once per process) and moves an open generation into replay — while a gap is open nothing is claimable, so the probe is the only signal — but recovery no longer touches the window: the recovery notice's own retries are the write-health probe and gate the backlog until one lands, no token probe runs while replaying, and until a write succeeds the anchor keeps naming the same generation, so one outage never opens a second generation with a second notice. A genuine second outage during replay begins after that success and gets its own generation. Two definitions tightened alongside: a definite configuration rejection (wrong channel, bad token) is durable per-intent state and no longer anchors the outage window at all, and a generation opens on the FAILURE that lands at least five minutes into the window rather than on the mere absence of a success — a single failure followed by five quiet minutes is an ordinary delay. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- cmd/alertint/situation_notifications.go | 6 +- cmd/alertint/situation_slack_e2e_test.go | 161 +++++++++++++++++- internal/situation/notification_worker.go | 126 ++++++++------ .../situation/notification_worker_test.go | 94 +++++++++- internal/store/notification_gaps.go | 40 +++-- internal/store/notification_gaps_test.go | 53 ++++++ 6 files changed, 392 insertions(+), 88 deletions(-) diff --git a/cmd/alertint/situation_notifications.go b/cmd/alertint/situation_notifications.go index 81717a4..51a8bb4 100644 --- a/cmd/alertint/situation_notifications.go +++ b/cmd/alertint/situation_notifications.go @@ -110,9 +110,11 @@ func NewSituationDeliverer(delivererStore DelivererStore, api slackDeliveryAPI, return &SituationDeliverer{store: delivererStore, api: api, channel: channel, now: now} } -// Probe verifies Slack readiness (auth.test) — the readiness check Task 7's +// Probe verifies TOKEN readiness (auth.test) — the readiness check Task 7's // gap lifecycle drives before recovery replay and before reactivating -// configuration-blocked intents. +// configuration-blocked intents. It proves the token can reach Slack and +// nothing about chat.postMessage/chat.update, so the worker never treats a +// successful probe as delivery health (NotificationWorker.probe). func (d *SituationDeliverer) Probe(ctx context.Context) error { return d.api.AuthTest(ctx) } diff --git a/cmd/alertint/situation_slack_e2e_test.go b/cmd/alertint/situation_slack_e2e_test.go index 89e4eb8..8f1386a 100644 --- a/cmd/alertint/situation_slack_e2e_test.go +++ b/cmd/alertint/situation_slack_e2e_test.go @@ -250,8 +250,9 @@ type e2eFixture struct { // controller cycle runs under. Empty (the default) is "no floor". slackFloor model.InterruptionPriority - worker *situation.NotificationWorker - l2 *e2eAssessmentClient + deliverer *SituationDeliverer + worker *situation.NotificationWorker + l2 *e2eAssessmentClient } func newE2EFixture(t *testing.T) *e2eFixture { @@ -270,13 +271,22 @@ func newE2EFixture(t *testing.T) *e2eFixture { }) deliverer := NewSituationDeliverer(st, client, e2eChannel, clock.Now) - f := &e2eFixture{t: t, ctx: ctx, st: st, clock: clock, slack: fake, l2: &e2eAssessmentClient{}} - f.worker = situation.NewNotificationWorker(st, deliverer, - situation.NotificationWorkerConfig{Owner: e2eOwner + ":notify"}, clock.Now, - slog.New(slog.DiscardHandler)) + f := &e2eFixture{t: t, ctx: ctx, st: st, clock: clock, slack: fake, deliverer: deliverer, l2: &e2eAssessmentClient{}} + f.restartWorker() return f } +// restartWorker replaces the notification worker with a fresh one over the +// same Store and deliverer — what a process restart with corrected +// configuration looks like to the durable ledger (the worker applies its +// configuration correction exactly once per process). +func (f *e2eFixture) restartWorker() { + f.t.Helper() + f.worker = situation.NewNotificationWorker(f.st, f.deliverer, + situation.NotificationWorkerConfig{Owner: e2eOwner + ":notify"}, f.clock.Now, + slog.New(slog.DiscardHandler)) +} + // e2eAssessmentClient answers every L2 dispatch with one accepted, schema- // valid proposal. attention/claimReason steer the derived Operator contract // exactly the way internal/situation's own replay fixture does. @@ -1308,3 +1318,142 @@ func TestSituationSlackE2ESupersededSuccessfulFirstPostDoesNotCreateSecondRoot(t t.Fatalf("%d effect(s) still owed once the queue drained%s", remaining, f.intentSummary()) } } + +// ---------------------------------------------------------------------- +// 10. A successful auth.test is token readiness, not delivery health: with +// writes failing continuously the five-minute Delivery gap still opens +// (review round 1, R1-F4). +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EAuthSuccessDoesNotEraseContinuousWriteFailure(t *testing.T) { + f := newE2EFixture(t) + f.seed("group=e2e-write-outage") + f.slack.setScript(func(method string, _ *e2eSlackCall) e2eSlackReply { + if method == "auth.test" { + return e2eSlackReply{} + } + return e2eSlackReply{HTTPStatus: http.StatusServiceUnavailable} + }) + for i := 0; i < 8; i++ { + f.deliverRound() + f.clock.advance(time.Minute) + } + failed := 0 + for _, c := range f.slack.snapshot() { + if c.Method == "chat.postMessage" && !c.Accepted { + failed++ + } + } + if failed < 2 { + t.Fatal("fixture did not repeatedly fail actual writes") + } + if n := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); n == 0 { + t.Fatalf("%d write failures over 8 minutes but no Delivery gap; a successful auth.test must not reset the failure anchor", failed) + } +} + +// ---------------------------------------------------------------------- +// 12. A recovery notice that keeps failing holds the backlog and opens no +// second generation for the same outage; the notice is the write-health +// probe (review round 1, R1-F4). +// ---------------------------------------------------------------------- + +func TestSituationSlackE2ERecoveryNoticeFailureHoldsTheBacklog(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysStatus(http.StatusServiceUnavailable, 0)) + sitID := f.seed("group=e2e-notice-fails") + f.deliverRound() + for elapsed := time.Duration(0); elapsed < 6*time.Minute; elapsed += time.Minute { + f.clock.advance(time.Minute) + f.deliverRound() + } + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 1 { + t.Fatalf("delivery gap generations = %d, want 1%s", gaps, f.intentSummary()) + } + + // The token comes back but writes do not: the probe recovers the + // generation, and the recovery notice then fails on every attempt. + f.slack.setScript(func(method string, _ *e2eSlackCall) e2eSlackReply { + if method == "auth.test" { + return e2eSlackReply{} + } + return e2eSlackReply{HTTPStatus: http.StatusServiceUnavailable} + }) + for i := 0; i < 12; i++ { + f.clock.advance(time.Minute) + f.deliverRound() + } + if got := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps WHERE status = 'replaying'`); got != 1 { + t.Fatalf("replaying generations = %d, want the recovered one%s", got, f.intentSummary()) + } + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 1 { + t.Fatalf("delivery gap generations = %d after twelve more failing minutes, want still 1: the same outage never opens a second generation%s", gaps, f.intentSummary()) + } + if accepted := len(f.slack.accepted()); accepted != 0 { + t.Fatalf("%d message(s) reached Slack while the recovery notice was still failing; the notice must precede the backlog", accepted) + } + notices := f.intentsOfClass("installation_gap_recovery") + if len(notices) != 1 || notices[0].Status != "pending" { + t.Fatalf("recovery notices = %+v, want exactly one, still pending (retrying indefinitely)", notices) + } + + // Writes come back: the notice lands first, then the backlog replays + // in order and the generation completes. + f.slack.setScript(alwaysOK) + f.clock.advance(6 * time.Minute) + f.deliverUntilQuiet(40) + calls := f.slack.accepted() + if len(calls) == 0 || !strings.Contains(calls[0].Text, "AlertINT's Slack delivery was interrupted") { + t.Fatalf("the first accepted message is not the recovery notice; accepted = %d", len(calls)) + } + _, rootTS := f.rootCoordinates(sitID) + if rootTS == "" { + t.Fatal("the Situation's root never published after the notice landed") + } + assertJournalRepliesInSequenceOrder(t, f, sitID, rootTS) + if got := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps WHERE status = 'complete'`); got != 1 { + t.Fatalf("complete generations = %d, want 1%s", got, f.intentSummary()) + } + if failing := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_state WHERE first_failure_at IS NOT NULL`); failing != 0 { + t.Fatal("the failure window is still open after deliveries succeeded") + } +} + +// ---------------------------------------------------------------------- +// 13. A blocked channel with a valid token is configuration, not an +// outage: effects block, no Delivery gap ever opens, and correcting +// the configuration delivers them (review round 1, R1-F4). +// ---------------------------------------------------------------------- + +func TestSituationSlackE2EBlockedChannelOpensNoGap(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(func(method string, _ *e2eSlackCall) e2eSlackReply { + if method == "auth.test" { + return e2eSlackReply{} + } + return e2eSlackReply{ErrorCode: "channel_not_found"} + }) + f.seed("group=e2e-blocked-channel") + for i := 0; i < 8; i++ { + f.deliverRound() + f.clock.advance(time.Minute) + } + if blocked := f.scalarInt(`SELECT COUNT(*) FROM notification_intents WHERE status = 'blocked_configuration'`); blocked == 0 { + t.Fatalf("no intent reached blocked_configuration%s", f.intentSummary()) + } + if gaps := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_gaps`); gaps != 0 { + t.Fatalf("delivery gap generations = %d, want 0: a configuration rejection is not a Slack outage%s", gaps, f.intentSummary()) + } + if failing := f.scalarInt(`SELECT COUNT(*) FROM slack_delivery_state WHERE first_failure_at IS NOT NULL`); failing != 0 { + t.Fatal("a configuration rejection anchored the continuous-failure window") + } + // With a valid token the worker already applied its one per-process + // configuration correction when the blocks appeared; corrected + // configuration means a restart. + f.slack.setScript(alwaysOK) + f.restartWorker() + f.deliverUntilQuiet(12) + if remaining := len(f.pendingBesidesDelivered()); remaining != 0 { + t.Fatalf("%d intent(s) still owed after corrected configuration%s", remaining, f.intentSummary()) + } +} diff --git a/internal/situation/notification_worker.go b/internal/situation/notification_worker.go index 1eb6471..256c837 100644 --- a/internal/situation/notification_worker.go +++ b/internal/situation/notification_worker.go @@ -80,8 +80,10 @@ type NotificationDelivery struct { type SlackDeliveryState struct { // FirstFailureAt anchors the current CONTINUOUS failure window. Nil // means Slack delivery is currently healthy. It is set by the first - // retryable/configuration failure and cleared by any success — it never - // slides forward while failures continue. + // retryable/uncertain delivery failure and cleared only by an actual + // delivery success — never by a readiness probe, which proves the token + // and nothing else — and it never slides forward while failures + // continue. FirstFailureAt *time.Time LastSuccessAt *time.Time // OpenGapGeneration is the generation currently open or replaying, nil @@ -173,22 +175,20 @@ const ( DeliveryInvalid DeliveryErrorClass = "invalid" ) -// isSlackOutcome reports whether this class is evidence about the Slack -// dependency itself. Only a real transport/API result is: an invalid -// payload is this build's own bug, and a local data-state condition never -// reached the wire. Neither may open a Delivery gap or hold the -// continuous-failure window open (spec.md's gap lifecycle is defined over -// "Slack delivery failure", not over every failed attempt). -func (c DeliveryErrorClass) isSlackOutcome() bool { - switch c { - case DeliveryInvalid, DeliveryLocalRetryable: - return false - case DeliveryRetryable, DeliveryConfigurationBlocking: - return true - default: - // An unknown class is not proof that Slack answered. - return false - } +// anchorsFailureWindow reports whether this class is evidence of a Slack +// OUTAGE — what the continuous-failure window and the Delivery-gap +// lifecycle are defined over (spec.md: "A first retryable or uncertain +// Slack delivery failure is an ordinary delay ... If failures remain +// continuous for five minutes, AlertINT opens one durable Delivery-gap +// generation"). Only a retryable or uncertain transport/API result is: an +// invalid payload is this build's own bug, a local data-state condition +// never reached the wire, and a definite configuration rejection is +// durable per-intent state (`blocked_configuration`, visible as the blocked +// backlog) rather than an outage — a wrong channel with a valid token is +// not Slack being down, and must never open a gap or hold one open +// (review round 1, R1-F4). +func (c DeliveryErrorClass) anchorsFailureWindow() bool { + return c == DeliveryRetryable } // DeliveryFailure is the classification a deliverer error may carry. An @@ -565,29 +565,15 @@ func (w *NotificationWorker) RunOnce(ctx context.Context) (int, error) { return handled, nil } -// advanceGapState drives the whole gap lifecycle for one round: probe while -// a failure window or gap exists, open a generation once failures have been -// continuous for the threshold, recover it once Slack answers again, and -// complete it once nothing replayable remains. +// advanceGapState drives the gap lifecycle for one round: probe while a +// failure window or gap exists, recover an open generation once the token +// answers again, and complete a replaying one once nothing replayable +// remains. Opening a generation is not a per-round question — it happens in +// observeFailure, on the failure that proves the window continuous. func (w *NotificationWorker) advanceGapState(ctx context.Context, state SlackDeliveryState, now time.Time) { if w.shouldProbe(state, now) { w.probe(ctx, state, now) } - if state.FirstFailureAt != nil { - opened, err := w.store.OpenDueDeliveryGap(ctx, now, w.cfg.GapThreshold) - if err != nil { - w.logger.Error("situation: notification worker: open delivery gap failed", "err", err) - } else if opened { - w.count(func(s *NotificationWorkerStats) { s.GapsOpened++ }) - w.auditAppend(ctx, auditKindGapOpened, map[string]any{ - "first_failure_at": state.FirstFailureAt.UTC().Format(time.RFC3339Nano), - "continuous_for_ms": now.Sub(*state.FirstFailureAt).Milliseconds(), - }) - w.logger.Warn("situation: notification worker: slack delivery gap opened", - "first_failure_at", state.FirstFailureAt.Format(time.RFC3339), - "continuous_for", now.Sub(*state.FirstFailureAt).String()) - } - } // Bounded: at most one completion per round keeps this cheap, and a // second finished generation completes on the next tick. if generation, done, err := w.store.CompleteDeliveryGap(ctx, now); err != nil { @@ -610,24 +596,41 @@ func (w *NotificationWorker) shouldProbe(state SlackDeliveryState, now time.Time if !w.probedOnce { return true } - // A REPLAYING generation is a recovered one: its own deliveries prove - // Slack health. Durably blocked configuration justifies probing only - // until this process has applied its startup correction — after that a - // probe can no longer change the outcome (only a restart with corrected + // Durably blocked configuration justifies probing only until this + // process has applied its startup correction — after that a probe can + // no longer change the outcome (only a restart with corrected // configuration can), and probing on it forever is the treadmill this // worker must not run. blockedStillMatters := state.BlockedConfigurationCount > 0 && !w.configurationReactivated.Load() if state.FirstFailureAt == nil && state.OpenGapStatus != "open" && !blockedStillMatters { return false } + // While a generation is REPLAYING, the recovery notice's own retries + // are the write-health probe (they gate the backlog until one lands), + // and a token probe could change nothing: it never clears the window. + if state.OpenGapStatus == "replaying" && !blockedStillMatters { + return false + } wait := notificationRetryDelay(w.probeFailures, 0, w.cfg.RetryInitial, w.cfg.RetryMax, 0, 0) return !now.Before(w.lastProbeAt.Add(wait)) } -// probe asks the deliverer whether Slack is usable and applies the outcome: -// a success clears the failure window, reactivates configuration-blocked -// intents, and moves any open gap into replay; a failure keeps the window -// continuous. +// probe asks the deliverer whether Slack is reachable with this +// installation's token and applies the outcome: a success reactivates +// configuration-blocked intents (once per process) and moves any open gap +// into replay; a retryable failure keeps the window continuous. +// +// A successful probe is TOKEN readiness, not delivery health: Probe is +// auth.test, which says nothing about whether chat.postMessage or +// chat.update work. It therefore never clears the continuous-failure +// window — only an actual delivery success does (acknowledgeDelivered). +// Otherwise a partial outage with auth healthy would reset the window on +// every probe, the mandatory five-minute gap could never open, and a gap +// could be declared recovered while writes still fail (review round 1, +// R1-F4). Moving an open gap into replay on a probe is still right: while +// a gap is open nothing is claimable, so the probe is the only signal; +// once replaying, the recovery notice's own retries are the write-health +// probe and gate the backlog until one lands. func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState, now time.Time) { w.mu.Lock() w.probedOnce = true @@ -641,7 +644,7 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState w.mu.Unlock() w.count(func(s *NotificationWorkerStats) { s.ProbeFailures++ }) class, code, _ := classifyDeliveryFailure(err) - if class.isSlackOutcome() { + if class.anchorsFailureWindow() { w.observeFailure(ctx, state, code, now) } return @@ -650,9 +653,6 @@ func (w *NotificationWorker) probe(ctx context.Context, state SlackDeliveryState w.mu.Lock() w.probeFailures = 0 w.mu.Unlock() - if err := w.store.ObserveSlackSuccess(ctx, now); err != nil { - w.logger.Error("situation: notification worker: record slack success failed", "err", err) - } if _, err := w.ReactivateConfiguration(ctx); err != nil { w.logger.Error("situation: notification worker: reactivate configuration-blocked intents failed", "err", err) } @@ -725,6 +725,13 @@ func (w *NotificationWorker) ReactivateConfiguration(ctx context.Context) (int, // observeFailure records one Slack failure against the continuous window and // emits the bounded WARNs the console action trail expects: one on the first // failure of a window, then paced retry WARNs — never one per attempt. +// observeFailure records one outage-evidence failure (anchorsFailureWindow) +// and, when it lands at least the gap threshold after the window's anchor +// with no delivery success in between, opens the Delivery-gap generation. +// spec.md: "If failures remain continuous for five minutes" — continuity +// is proven by a FAILURE that far into the window, never inferred from the +// absence of a success: a single failure followed by five quiet minutes +// (a retry not yet due, a process that was down) is an ordinary delay. func (w *NotificationWorker) observeFailure(ctx context.Context, state SlackDeliveryState, code string, now time.Time) { if err := w.store.ObserveSlackFailure(ctx, code, now); err != nil { w.logger.Error("situation: notification worker: record slack failure failed", "err", err) @@ -738,6 +745,18 @@ func (w *NotificationWorker) observeFailure(ctx context.Context, state SlackDeli "error_class", code) return } + if opened, err := w.store.OpenDueDeliveryGap(ctx, now, w.cfg.GapThreshold); err != nil { + w.logger.Error("situation: notification worker: open delivery gap failed", "err", err) + } else if opened { + w.count(func(s *NotificationWorkerStats) { s.GapsOpened++ }) + w.auditAppend(ctx, auditKindGapOpened, map[string]any{ + "first_failure_at": state.FirstFailureAt.UTC().Format(time.RFC3339Nano), + "continuous_for_ms": now.Sub(*state.FirstFailureAt).Milliseconds(), + }) + w.logger.Warn("situation: notification worker: slack delivery gap opened", + "first_failure_at", state.FirstFailureAt.Format(time.RFC3339), + "continuous_for", now.Sub(*state.FirstFailureAt).String()) + } w.mu.Lock() due := now.Sub(w.lastWarnAt) >= notificationWarnCadence if due { @@ -880,11 +899,10 @@ func (w *NotificationWorker) acknowledgeFailure(ctx context.Context, claim Notif if stateErr != nil { w.logger.Error("situation: notification worker: read slack delivery state failed", "err", stateErr) } - if class.isSlackOutcome() { - // Only a real Slack answer moves the dependency-health window. An - // invalid payload is this build's own bug, and a local data-state - // condition never reached the wire; neither may open a Delivery - // gap or report an outage that is not happening. + if class.anchorsFailureWindow() { + // Only a retryable/uncertain Slack answer moves the dependency- + // health window (anchorsFailureWindow): nothing else may open a + // Delivery gap or report an outage that is not happening. w.observeFailure(ctx, state, code, now) } diff --git a/internal/situation/notification_worker_test.go b/internal/situation/notification_worker_test.go index 7e263ae..69f0a9d 100644 --- a/internal/situation/notification_worker_test.go +++ b/internal/situation/notification_worker_test.go @@ -436,8 +436,8 @@ func TestNotificationWorkerBlocksConfigurationAndFailsInvalid(t *testing.T) { if len(s.retried) != 0 { t.Fatalf("retried = %v, want neither outcome to schedule a retry", s.retried) } - if len(s.failures) != 1 || s.failures[0] != "invalid_auth" { - t.Fatalf("observed slack failures = %v, want only the configuration rejection", s.failures) + if len(s.failures) != 0 { + t.Fatalf("observed slack failures = %v, want none: a configuration rejection is durable per-intent state, not an outage", s.failures) } }) } @@ -626,9 +626,23 @@ func TestNotificationWorkerProbesWhileAFailureWindowExists(t *testing.T) { if deliverer.probeCalls != 2 { t.Fatalf("probe calls with an open failure window = %d, want 2", deliverer.probeCalls) } + store.snapshot(func(s *nwStore) { + // No failure landed in any of these rounds (every probe + // succeeded), so nothing proved the window continuous: a gap is + // opened by a failure, never by the passage of time. + if len(s.gapOpens) != 0 { + t.Fatalf("OpenDueDeliveryGap calls = %d, want 0 without a fresh failure", len(s.gapOpens)) + } + }) + // A fresh retryable failure inside the window asks the store. + deliverer.probeErr = errors.New("slack unreachable") + clock.advance(time.Minute) + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("fifth RunOnce: %v", err) + } store.snapshot(func(s *nwStore) { if len(s.gapOpens) == 0 { - t.Fatal("a failure window must ask the store whether a gap is due") + t.Fatal("a failure inside an open window must ask the store whether a gap is due") } if s.gapOpens[len(s.gapOpens)-1] != defaultDeliveryGapThreshold { t.Fatalf("gap threshold = %s, want the fixed %s", s.gapOpens[len(s.gapOpens)-1], defaultDeliveryGapThreshold) @@ -637,9 +651,10 @@ func TestNotificationWorkerProbesWhileAFailureWindowExists(t *testing.T) { } // TestNotificationWorkerRecoveryReactivatesConfigurationAndReplaysGap -// proves a successful probe closes the failure window, increments the -// durable configuration generation for blocked intents, and recovers the -// open gap — in that order, before any backlog claim. +// proves a successful probe increments the durable configuration generation +// for blocked intents and recovers the open gap — in that order, before any +// backlog claim — and never closes the failure window: auth.test is token +// readiness, not delivery health (review round 1, R1-F4). func TestNotificationWorkerRecoveryReactivatesConfigurationAndReplaysGap(t *testing.T) { now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) failedAt := now.Add(-10 * time.Minute) @@ -660,8 +675,8 @@ func TestNotificationWorkerRecoveryReactivatesConfigurationAndReplaysGap(t *test t.Fatalf("RunOnce: %v", err) } store.snapshot(func(s *nwStore) { - if s.successes == 0 { - t.Fatal("a successful probe must close the failure window") + if s.successes != 0 { + t.Fatal("a successful probe closed the failure window; only a delivery success may") } if len(s.reactivated) != 1 || s.reactivated[0] != 4 { t.Fatalf("reactivated with generations %v, want exactly the incremented [4]", s.reactivated) @@ -676,6 +691,69 @@ func TestNotificationWorkerRecoveryReactivatesConfigurationAndReplaysGap(t *test } } +// TestNotificationWorkerProbeSuccessNeverClearsTheFailureWindow pins the +// partial-outage case directly: auth.test keeps succeeding while every +// write fails, and the window still reaches the gap threshold. +func TestNotificationWorkerProbeSuccessNeverClearsTheFailureWindow(t *testing.T) { + clock := &nwClock{at: time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC)} + failedAt := clock.at + store := &nwStore{state: SlackDeliveryState{FirstFailureAt: &failedAt}} + for i := 0; i < 8; i++ { + store.batches = append(store.batches, []NotificationClaim{nwClaim(fmt.Sprintf("intent-%d", i), i+1)}) + } + deliverer := &nwDeliverer{deliver: func(model.NotificationIntent) (NotificationDelivery, error) { + return NotificationDelivery{}, nwDeliveryError{class: DeliveryRetryable, code: "http_503"} + }} + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", Heartbeat: time.Hour, Rand: func() float64 { return 0.5 }, + }, clock.now, nwLogger()) + for i := 0; i < 8; i++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", i, err) + } + clock.advance(time.Minute) + } + if deliverer.probeCalls < 2 { + t.Fatalf("probe calls = %d, want the worker to keep probing while the window is open", deliverer.probeCalls) + } + store.snapshot(func(s *nwStore) { + if s.successes != 0 { + t.Fatalf("ObserveSlackSuccess calls = %d, want 0: %d successful token probes are not delivery successes", s.successes, deliverer.probeCalls) + } + if len(s.failures) != 8 { + t.Fatalf("observed slack failures = %d, want one per failed write", len(s.failures)) + } + if len(s.gapOpens) == 0 { + t.Fatal("the failure window never asked the store whether a gap is due") + } + }) +} + +// TestNotificationWorkerDoesNotProbeWhileReplaying proves that once a +// generation is replaying the recovery notice's own retries are the +// write-health probe: no token probe runs (it could change nothing). +func TestNotificationWorkerDoesNotProbeWhileReplaying(t *testing.T) { + clock := &nwClock{at: time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC)} + failedAt := clock.at.Add(-10 * time.Minute) + generation := "gap-1" + store := &nwStore{state: SlackDeliveryState{ + FirstFailureAt: &failedAt, OpenGapGeneration: &generation, OpenGapStatus: "replaying", + }} + deliverer := &nwDeliverer{} + w := NewNotificationWorker(store, deliverer, NotificationWorkerConfig{ + Owner: "notify-a", Heartbeat: time.Hour, Rand: func() float64 { return 0.5 }, + }, clock.now, nwLogger()) + for i := 0; i < 3; i++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce %d: %v", i, err) + } + clock.advance(time.Minute) + } + if deliverer.probeCalls != 1 { + t.Fatalf("probe calls while replaying = %d, want exactly the one startup probe", deliverer.probeCalls) + } +} + // TestNotificationWorkerOpensAndCompletesGapGenerations proves the worker // drives both ends of the durable generation lifecycle and counts them. func TestNotificationWorkerOpensAndCompletesGapGenerations(t *testing.T) { diff --git a/internal/store/notification_gaps.go b/internal/store/notification_gaps.go index cf9b9e6..3f2ffab 100644 --- a/internal/store/notification_gaps.go +++ b/internal/store/notification_gaps.go @@ -26,9 +26,14 @@ import ( // open --successful readiness probe--> replaying (+ one recovery notice) // replaying --backlog drained--> complete // -// A success at any point clears the failure window, so only CONTINUOUS -// failure ever opens a gap. Generations are never deleted and never merged: -// a second outage during replay gets its own identity after its own full +// A DELIVERY success at any point clears the failure window, so only +// CONTINUOUS failure ever opens a gap. A readiness probe does not: it +// authorizes replay (the recovery notice becomes claimable) but proves only +// the token, so the window it found stays anchored until a write actually +// lands — while the notice keeps failing, the anchor keeps naming the same +// generation and no second one opens for the same outage. Generations are +// never deleted and never merged: a second outage during replay — failures +// after a real success — gets its own identity after its own full // five-minute window. // // Which intents a generation is replaying is DERIVED, not stored: migration @@ -251,17 +256,21 @@ func (s *Store) OpenDueDeliveryGap(ctx context.Context, now time.Time, threshold } // RecoverDeliveryGap moves the oldest open generation into replay in one -// idempotent transaction: it closes the failure window, recomputes the -// backlog the notice reports, and creates the single claimable -// installation_gap_recovery intent that must deliver before any of that -// backlog does. It reports the generation it recovered. +// idempotent transaction: it recomputes the backlog the notice reports and +// creates the single claimable installation_gap_recovery intent that must +// deliver before any of that backlog does. It reports the generation it +// recovered. // -// Closing the window here as well as in ObserveSlackSuccess is deliberate, -// not redundant: recovery is by definition proof that Slack answered, and -// the next generation's identity is derived from the NEXT window's anchor -// (NewGapGenerationID). A caller that recovered without first clearing the -// old anchor would leave a second outage unable to open a generation of its -// own, because it would keep computing the completed generation's id. +// It deliberately does NOT close the failure window. The readiness check +// that drives it is auth.test — token readiness, not delivery health — so +// clearing the anchor here would let a gap be declared recovered while +// writes still fail, and a second generation would open five minutes later +// for the same outage, with a second notice, and so on (review round 1, +// R1-F4). The anchor is cleared by ObserveSlackSuccess when a delivery — +// the recovery notice first of all — actually lands; until then a new +// failure keeps computing this generation's id (NewGapGenerationID) and +// OpenDueDeliveryGap opens nothing. A genuine second outage begins after +// that success, from a fresh anchor, and gets its own generation. func (s *Store) RecoverDeliveryGap(ctx context.Context, now time.Time) (string, bool, error) { now = now.UTC() nowStr := canonicalTime(now) @@ -309,11 +318,6 @@ func (s *Store) RecoverDeliveryGap(ctx context.Context, now time.Time) (string, nowStr, affected, delayed, notice.ID, generation); err != nil { return "", false, fmt.Errorf("store: move delivery gap into replay: %w", err) } - if _, err := tx.ExecContext(ctx, ` - UPDATE slack_delivery_state SET first_failure_at = NULL, last_success_at = ?, updated_at = ? - WHERE id = 1`, nowStr, nowStr); err != nil { - return "", false, fmt.Errorf("store: close the recovered failure window: %w", err) - } if err := tx.Commit(); err != nil { return "", false, fmt.Errorf("store: commit recover delivery gap: %w", err) } diff --git a/internal/store/notification_gaps_test.go b/internal/store/notification_gaps_test.go index 1affd90..494476b 100644 --- a/internal/store/notification_gaps_test.go +++ b/internal/store/notification_gaps_test.go @@ -283,6 +283,11 @@ func TestDeliveryGapReplaySecondOutageStartsADistinctGeneration(t *testing.T) { if err != nil { t.Fatalf("RecoverDeliveryGap: %v", err) } + // The recovery notice lands: a real delivery success closes the first + // outage's window. + if err := st.ObserveSlackSuccess(ctx, now.Add(7*time.Minute+30*time.Second)); err != nil { + t.Fatalf("ObserveSlackSuccess: %v", err) + } // Slack fails again mid-replay. secondWindow := now.Add(8 * time.Minute) @@ -303,6 +308,54 @@ func TestDeliveryGapReplaySecondOutageStartsADistinctGeneration(t *testing.T) { } } +// TestDeliveryGapRecoveryKeepsTheFailureWindowUntilAWriteLands proves a +// readiness probe's recovery does not close the failure window: while the +// recovery notice keeps failing, the same outage keeps naming the same +// generation and no second one opens (review round 1, R1-F4). +func TestDeliveryGapRecoveryKeepsTheFailureWindowUntilAWriteLands(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + snSeedOneCycle(t, st, "group-gap-keep-window", now) + + snFail(t, st, now) + if _, err := st.OpenDueDeliveryGap(ctx, now.Add(snGapThreshold), snGapThreshold); err != nil { + t.Fatalf("OpenDueDeliveryGap: %v", err) + } + if _, ok, err := st.RecoverDeliveryGap(ctx, now.Add(7*time.Minute)); err != nil || !ok { + t.Fatalf("RecoverDeliveryGap = (%v, %v), want recovered", ok, err) + } + state := snState(t, st) + if state.FirstFailureAt == nil || !state.FirstFailureAt.Equal(now) { + t.Fatalf("first_failure_at after recovery = %v, want the original anchor %s: a token probe is not a delivery success", state.FirstFailureAt, now) + } + if state.OpenGapStatus != "replaying" { + t.Fatalf("gap status after recovery = %q, want replaying", state.OpenGapStatus) + } + // The notice keeps failing for well over five more minutes: same + // outage, same generation, no second one. + snFail(t, st, now.Add(8*time.Minute)) + snFail(t, st, now.Add(14*time.Minute)) + if opened, err := st.OpenDueDeliveryGap(ctx, now.Add(15*time.Minute), snGapThreshold); err != nil || opened { + t.Fatalf("OpenDueDeliveryGap while the recovered outage continues = (%v, %v), want (false, nil)", opened, err) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM slack_delivery_gaps`); n != 1 { + t.Fatalf("gap generations = %d, want exactly 1 for one continuous outage", n) + } + // The notice finally lands; the window closes and a LATER failure + // starts a fresh window of its own. + if err := st.ObserveSlackSuccess(ctx, now.Add(16*time.Minute)); err != nil { + t.Fatalf("ObserveSlackSuccess: %v", err) + } + if snState(t, st).FirstFailureAt != nil { + t.Fatal("a delivery success must close the failure window") + } + snFail(t, st, now.Add(17*time.Minute)) + if opened, err := st.OpenDueDeliveryGap(ctx, now.Add(17*time.Minute+snGapThreshold), snGapThreshold); err != nil || !opened { + t.Fatalf("OpenDueDeliveryGap for a second outage after a real success = (%v, %v), want (true, nil)", opened, err) + } +} + // TestDeliveryGapConfigurationGenerationReactivatesBlockedIntents proves // corrected startup configuration increments the durable configuration // generation and returns blocked intents to pending exactly once. From 99d9c79cc3f0a2c7fd4861678ee9dd39e870311f Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 14:01:01 +0300 Subject: [PATCH 25/31] feat(situation): recurrence milestones from durable occurrences The controller supplied only the count of prior terminal Situations as the recurrence count. With one nonterminal Situation per group that number is fixed for a Situation's whole lifetime, so the milestone comparison could never fire for re-fires attaching to the current Incident, and notify.slack.recurrence_mode reached only the retired legacy notifier. The docs had been corrected to say the setting has no effect, but the spec still requires preserving recurrence configuration and milestones in the owning Situation thread (review round 1, R1-F6). The snapshot now carries each member Incident's recurrence-collapse occurrence count from the Store; the recurrence count is prior terminal Situations plus those occurrences, so a re-fire's membership input can cross a milestone rung and produce the recurrence_milestone Transition (a quiet thread entry, never a poke, per the existing planner rule). notify.slack.recurrence_mode reaches the planner through the controller config: change-gated posts the milestone reply, off keeps only the silent root edit. Quiet Situations still have no Slack recurrence trace. The public docs and the drill CLI messages now describe this instead of the "no effect" claim. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- cmd/alertint/drill.go | 13 +- cmd/alertint/main.go | 2 +- cmd/alertint/situation_controller.go | 13 +- cmd/alertint/situation_controller_test.go | 13 +- cmd/alertint/situation_slack_e2e_test.go | 138 ++++++++++++++++++- docs/concepts/architecture.md | 8 +- docs/concepts/incident-memory.md | 11 +- docs/getting-started/configuration.md | 16 +-- docs/notifications/slack.md | 22 +-- internal/situation/controller.go | 38 ++++- internal/situation/controller_test.go | 20 +++ internal/situation/history_replay_test.go | 20 +-- internal/situation/notification_plan.go | 14 +- internal/situation/notification_plan_test.go | 46 +++++++ internal/situation/snapshot.go | 10 ++ internal/store/situation_controller.go | 9 +- internal/store/situation_controller_test.go | 36 +++++ 17 files changed, 369 insertions(+), 60 deletions(-) diff --git a/cmd/alertint/drill.go b/cmd/alertint/drill.go index 03dc767..fe324ef 100644 --- a/cmd/alertint/drill.go +++ b/cmd/alertint/drill.go @@ -249,7 +249,7 @@ func (d *drillCmd) run(ctx context.Context) error { return err } } else { - d.printf("fired the rerun; mcp is not usable from here — the occurrence count is visible over mcp or in the incidents table (an attach to a live Situation posts nothing to Slack).") + d.printf("fired the rerun; mcp is not usable from here — the occurrence count is visible over mcp or in the incidents table, and the owning Situation's root shows it as recurred ×N once it next edits.") } return d.maybeResolve(ctx, run, recvBase, webhookToken) } @@ -499,9 +499,10 @@ func (d *drillCmd) fetchDrillCandidates(ctx context.Context, mcpEndpoint, mcpTok // pollOccurrenceRerun polls the matched incident until its occurrence count // registers the collapsed re-fire, then prints the "recurred ×N" payoff. It -// exits as soon as the count increments; the count is the whole payoff, since -// an attach to a live Situation creates no Transition and therefore no Slack -// effect of any kind. +// exits as soon as the count increments. The attach feeds the owning +// Situation's recurrence count; only a crossed milestone rung creates a +// Transition (and one quiet thread reply), so a single rerun usually shows +// nothing new in Slack until the root next edits. func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToken, incidentID string) error { client := newMCPOneShotClient(mcpEndpoint, mcpToken, d.http) if err := client.initialize(ctx); err != nil { @@ -520,7 +521,7 @@ func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToke Occurrences int `json:"occurrences"` } if json.Unmarshal(raw, &p) == nil && p.Occurrences > 0 { - d.printf("collapsed: incident %s recurred ×%d — no second triage, and no new Slack message", incidentID, p.Occurrences+1) + d.printf("collapsed: incident %s recurred ×%d — no second triage; the owning Situation's recurrence count moved", incidentID, p.Occurrences+1) return nil } } @@ -530,7 +531,7 @@ func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToke } } } - d.printf("the occurrence has not registered yet; re-run with --result %s (an attach to a live Situation posts nothing to Slack, so there is no card edit to watch for)", incidentID) + d.printf("the occurrence has not registered yet; re-run with --result %s (the attach shows up as the owning Situation's recurrence count, not as a card edit of its own)", incidentID) return nil } diff --git a/cmd/alertint/main.go b/cmd/alertint/main.go index 3e5801d..d194ba3 100644 --- a/cmd/alertint/main.go +++ b/cmd/alertint/main.go @@ -452,7 +452,7 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error { // dependency of its own — corCfg/correlator.New's signature carries none, // and its only path to Acute Triage is via incidentSink{skill: skill}. crt, err := buildControllerRuntime(st, llmClient, llmHealth, skill, cfg.Situations, - cfg.Notify.Slack.MinSeverity, owner, auditor, logger) + cfg.Notify.Slack.MinSeverity, cfg.Notify.Slack.RecurrenceMode, owner, auditor, logger) if err != nil { return err } diff --git a/cmd/alertint/situation_controller.go b/cmd/alertint/situation_controller.go index d7319b8..9fdb41d 100644 --- a/cmd/alertint/situation_controller.go +++ b/cmd/alertint/situation_controller.go @@ -75,6 +75,7 @@ func newControllerRuntime( skill *acutetriage.Skill, cfg config.SituationsConfig, slackMinSeverity string, + recurrenceMode string, owner string, auditSink situation.AuditSink, logger *slog.Logger, @@ -82,7 +83,7 @@ func newControllerRuntime( if strings.TrimSpace(owner) == "" { panic("cmd/alertint: controller runtime requires a non-empty owner") } - controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, slackInterruptionFloor(slackMinSeverity), owner) + controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, slackInterruptionFloor(slackMinSeverity), recurrenceMode, owner) worker := situation.NewControllerWorker(st, st, assessClient, controllerCfg, workerCfg, nil, auditSink, logger) @@ -119,6 +120,7 @@ func buildControllerRuntime( skill *acutetriage.Skill, cfg config.SituationsConfig, slackMinSeverity string, + recurrenceMode string, owner string, auditSink situation.AuditSink, logger *slog.Logger, @@ -127,7 +129,7 @@ func buildControllerRuntime( if err != nil { return nil, fmt.Errorf("situation controller: %w", err) } - crt := newControllerRuntime(st, assessClient, skill, cfg, slackMinSeverity, owner, auditSink, logger) + crt := newControllerRuntime(st, assessClient, skill, cfg, slackMinSeverity, recurrenceMode, owner, auditSink, logger) crt.SetDependencyRecoveryWaker(llmHealthDependencyWaker{tracker: llmHealth, st: st}) crt.SetAssessmentHealthObserver(llmHealthAssessmentObserver{tracker: llmHealth}) return crt, nil @@ -153,12 +155,15 @@ func buildControllerRuntime( // situations.slack.repage_cooldown_seconds -> ControllerConfig. // RepageCooldown. Left at their zero values they would silently mean "no // floor" and "the built-in 900s default", so an operator who configured -// either would have been ignored. +// either would have been ignored. notify.slack.recurrence_mode -> +// ControllerConfig.RecurrenceMode carries the preserved recurrence +// configuration into Situation planning (review round 1, R1-F6). func situationsConfigToControllerConfig(cfg config.SituationsConfig, slackFloor model.InterruptionPriority, - owner string) (situation.ControllerConfig, situation.ControllerWorkerConfig) { + recurrenceMode, owner string) (situation.ControllerConfig, situation.ControllerWorkerConfig) { controllerCfg := situation.ControllerConfig{ SlackFloor: slackFloor, RepageCooldown: time.Duration(cfg.Slack.RepageCooldownSeconds) * time.Second, + RecurrenceMode: strings.ToLower(strings.TrimSpace(recurrenceMode)), Cadence: situation.CadenceTempo{ Fast: time.Duration(cfg.Cadence.FastSeconds) * time.Second, Normal: time.Duration(cfg.Cadence.NormalSeconds) * time.Second, diff --git a/cmd/alertint/situation_controller_test.go b/cmd/alertint/situation_controller_test.go index ca7d0d2..2a3fb0f 100644 --- a/cmd/alertint/situation_controller_test.go +++ b/cmd/alertint/situation_controller_test.go @@ -27,7 +27,7 @@ func TestSituationControllerRuntimePanicsOnEmptyOwner(t *testing.T) { t.Fatal("expected a panic for an empty owner") } }() - newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", " ", nil, nil) + newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "", " ", nil, nil) } // TestSituationsConfigToControllerConfigMapsEveryField pins Task 8's own @@ -55,7 +55,7 @@ func TestSituationsConfigToControllerConfigMapsEveryField(t *testing.T) { }, } cfg.Slack = config.SituationSlackConfig{RepageCooldownSeconds: 600} - controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, model.InterruptionHigh, "owner-1") + controllerCfg, workerCfg := situationsConfigToControllerConfig(cfg, model.InterruptionHigh, " Off ", "owner-1") // Plan 3 Task 9: the two publication-policy fields Task 5 added but left // unwired must carry real operator configuration, not their zero values @@ -66,6 +66,9 @@ func TestSituationsConfigToControllerConfigMapsEveryField(t *testing.T) { if controllerCfg.RepageCooldown != 600*time.Second { t.Fatalf("RepageCooldown = %v, want 600s from situations.slack.repage_cooldown_seconds", controllerCfg.RepageCooldown) } + if controllerCfg.RecurrenceMode != situation.RecurrenceModeOff { + t.Fatalf("RecurrenceMode = %q, want the normalized notify.slack.recurrence_mode %q", controllerCfg.RecurrenceMode, situation.RecurrenceModeOff) + } if controllerCfg.MaxL2CallsPerAttempt != 2 || controllerCfg.MaxWorkAttemptsPerInput != 5 { t.Fatalf("controllerCfg budgets = %+v", controllerCfg) @@ -135,7 +138,7 @@ func TestSituationControllerRuntimeSlackFloorMapsMinSeverity(t *testing.T) { func TestSituationControllerRuntimeRecoverAndBackfillOnEmptyStoreIsANoOp(t *testing.T) { st := newTestFoundationStore(t) - rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "test-owner", nil, nil) + rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "", "test-owner", nil, nil) report, err := rt.RecoverAndBackfill(context.Background(), time.Now().UTC()) if err != nil { @@ -150,7 +153,7 @@ func TestSituationControllerRuntimeRecoverAndBackfillOnEmptyStoreIsANoOp(t *test func TestSituationControllerRuntimeStartDrainStop(t *testing.T) { st := newTestFoundationStore(t) cfg := config.SituationsConfig{ReconcilePollSeconds: 3600, LeaseSeconds: 300, HeartbeatSeconds: 30} - rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, cfg, "", "test-owner", nil, nil) + rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, cfg, "", "", "test-owner", nil, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -538,7 +541,7 @@ func TestSituationControllerRuntimeRecoverAndBackfillAuditsStartupHorizonExhaust } audit := &fakeControllerRuntimeAuditSink{} - rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "test-owner", audit, nil) + rt := newControllerRuntime(st, &fakeOneShotClient{}, nil, config.SituationsConfig{}, "", "", "test-owner", audit, nil) report, err := rt.RecoverAndBackfill(context.Background(), now) if err != nil { diff --git a/cmd/alertint/situation_slack_e2e_test.go b/cmd/alertint/situation_slack_e2e_test.go index 8f1386a..994683a 100644 --- a/cmd/alertint/situation_slack_e2e_test.go +++ b/cmd/alertint/situation_slack_e2e_test.go @@ -249,6 +249,9 @@ type e2eFixture struct { // slackFloor is the operator's notify.slack.min_severity floor every // controller cycle runs under. Empty (the default) is "no floor". slackFloor model.InterruptionPriority + // recurrenceMode is the operator's notify.slack.recurrence_mode. Empty + // (the default) is change-gated. + recurrenceMode string deliverer *SituationDeliverer worker *situation.NotificationWorker @@ -408,7 +411,8 @@ func (f *e2eFixture) seedQuiet(groupKey string) string { func (f *e2eFixture) controllerCycle() int { f.t.Helper() f.clock.advance(time.Minute) - cw := situation.NewControllerWorker(f.st, f.st, f.l2, situation.ControllerConfig{SlackFloor: f.slackFloor}, + cw := situation.NewControllerWorker(f.st, f.st, f.l2, + situation.ControllerConfig{SlackFloor: f.slackFloor, RecurrenceMode: f.recurrenceMode}, situation.ControllerWorkerConfig{Owner: e2eOwner + ":controller", Now: f.clock.Now}, f.clock.Now, audit.New(f.st.DB()), slog.New(slog.DiscardHandler)) n, err := cw.Drain(f.ctx) @@ -1457,3 +1461,135 @@ func TestSituationSlackE2EBlockedChannelOpensNoGap(t *testing.T) { t.Fatalf("%d intent(s) still owed after corrected configuration%s", remaining, f.intentSummary()) } } + +// ---------------------------------------------------------------------- +// 14. Recurrence milestones come from durable Store facts and stay in the +// owning Situation thread: re-fires attaching as occurrences move the +// count, a crossed rung is one quiet thread entry (never a broadcast), +// and recurrence_mode: off keeps only the silent root edit +// (review round 1, R1-F6). +// ---------------------------------------------------------------------- + +// refire attaches one recurrence-collapse occurrence to the Situation's +// member Incident through the store's own occurrence path and enqueues +// the membership_changed Situation input a re-fire produces, then applies +// it — the same durable inputs ApplyCorrelatedDelivery leaves behind for a +// re-fire the Correlator collapsed onto a judged Incident. +func (f *e2eFixture) refire(groupKey string, n int) { + f.t.Helper() + incID := "inc-" + groupKey + for i := 0; i < n; i++ { + f.clock.advance(time.Minute) + now := f.clock.Now() + alertID := fmt.Sprintf("alert-%s-refire-%d", groupKey, i) + if _, err := f.st.DB().ExecContext(f.ctx, ` + INSERT INTO alerts (id, fingerprint, status, labels_json, annotations_json, starts_at, received_at) + VALUES (?, ?, 'firing', '{"alertname":"HighLatency"}', '{}', ?, ?)`, + alertID, "fp-"+alertID, now.UTC().Format(time.RFC3339Nano), now.UTC().Format(time.RFC3339Nano)); err != nil { + f.t.Fatalf("insert re-fire alert: %v", err) + } + if _, err := f.st.InsertOccurrenceAndAttach(f.ctx, store.Occurrence{ + IncidentID: incID, OccurredAt: now, Fingerprints: []string{"fp-" + alertID}, + Payload: []store.OccurrenceMember{}, TriggerKind: "none", + }, alertID, now); err != nil { + f.t.Fatalf("attach occurrence: %v", err) + } + inputID := fmt.Sprintf("input-%s-refire-%d", groupKey, i) + if _, err := f.st.DB().ExecContext(f.ctx, ` + INSERT INTO situation_input_outbox (id, idempotency_key, incident_id, kind, group_key, occurred_at, status) + VALUES (?, ?, ?, 'membership_changed', ?, ?, 'pending')`, + inputID, "idem:"+inputID, incID, groupKey, now.UTC().Format(time.RFC3339Nano)); err != nil { + f.t.Fatalf("insert re-fire situation input: %v", err) + } + claims, err := f.st.ClaimSituationInputs(f.ctx, "e2e-refire:"+inputID, now, time.Minute, 1) + if err != nil || len(claims) != 1 { + f.t.Fatalf("claim re-fire input: claims=%d err=%v", len(claims), err) + } + if err := f.st.ApplySituationInput(f.ctx, claims[0]); err != nil { + f.t.Fatalf("apply re-fire input: %v", err) + } + } +} + +func TestSituationSlackE2ERecurrenceMilestoneStaysInThread(t *testing.T) { + f := newE2EFixture(t) + f.slack.setScript(alwaysOK) + // seed's five prior episodes put the live Situation at recurrence 5 — + // the first rung — on its first Transition. + sitID := f.seed("group=e2e-milestone") + f.deliverUntilQuiet(12) + _, rootTS := f.rootCoordinates(sitID) + if rootTS == "" { + t.Fatal("the first root never published") + } + before := len(f.slack.accepted()) + + // Five re-fires attach as occurrences: the count reaches 10, the next + // rung. Four of the five cycles cross no rung and are non-material. + f.refire("group=e2e-milestone", 5) + if n := f.controllerCycle(); n == 0 { + t.Fatal("no controller work was due after the re-fires") + } + milestones := f.scalarInt(`SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ? AND reason = 'recurrence_milestone'`, sitID) + if milestones != 1 { + t.Fatalf("recurrence_milestone transitions = %d, want exactly 1 (the ×10 rung)%s", milestones, f.intentSummary()) + } + if count := f.scalarInt(`SELECT json_extract(summary_json,'$.recurrence_count') FROM situation_episode_summaries WHERE situation_id = ?`, sitID); count != 10 { + t.Fatalf("episode recurrence count = %d, want 10 (five prior Situations plus five occurrences)", count) + } + f.deliverUntilQuiet(12) + + var threadReplies, broadcasts, rootEdits int + for _, c := range f.slack.accepted()[before:] { + switch { + case c.Method == "chat.update": + rootEdits++ + case c.Method == "chat.postMessage" && c.ThreadTS == rootTS && !c.Broadcast: + threadReplies++ + case c.Method == "chat.postMessage" && (c.ThreadTS != rootTS || c.Broadcast): + broadcasts++ + } + } + if rootEdits != 1 || threadReplies != 1 || broadcasts != 0 { + t.Fatalf("milestone delivery = %d root edit(s), %d quiet thread reply(ies), %d channel message(s); want 1, 1, 0: a milestone stays in the owning thread and never re-pages%s", + rootEdits, threadReplies, broadcasts, f.intentSummary()) + } + if !strings.Contains(f.slack.accepted()[len(f.slack.accepted())-1].Text, "Recurrence milestone") { + t.Fatalf("the milestone reply does not render the milestone: %q", f.slack.accepted()[len(f.slack.accepted())-1].Text) + } +} + +func TestSituationSlackE2ERecurrenceModeOffEditsTheRootOnly(t *testing.T) { + f := newE2EFixture(t) + f.recurrenceMode = situation.RecurrenceModeOff + f.slack.setScript(alwaysOK) + sitID := f.seed("group=e2e-milestone-off") + f.deliverUntilQuiet(12) + _, rootTS := f.rootCoordinates(sitID) + if rootTS == "" { + t.Fatal("the first root never published") + } + before := len(f.slack.accepted()) + + f.refire("group=e2e-milestone-off", 5) + if n := f.controllerCycle(); n == 0 { + t.Fatal("no controller work was due after the re-fires") + } + if milestones := f.scalarInt(`SELECT COUNT(*) FROM situation_transitions WHERE situation_id = ? AND reason = 'recurrence_milestone'`, sitID); milestones != 1 { + t.Fatalf("recurrence_milestone transitions = %d, want 1: off never suppresses history%s", milestones, f.intentSummary()) + } + f.deliverUntilQuiet(12) + var rootEdits, posts int + for _, c := range f.slack.accepted()[before:] { + switch c.Method { + case "chat.update": + rootEdits++ + case "chat.postMessage": + posts++ + } + } + if rootEdits != 1 || posts != 0 { + t.Fatalf("recurrence_mode off delivered %d root edit(s) and %d post(s); want 1 and 0: only the silent count update%s", + rootEdits, posts, f.intentSummary()) + } +} diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index eb78ff1..415c812 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -192,10 +192,10 @@ Before spending an analysis, **AlertINT** checks whether it has seen this condition before. A re-fire of an already-analyzed group key inside the collapse horizon attaches as an **occurrence** — no second LLM call; a released binary edits the Incident card in place. On the `state-controller` -branch that attach leaves no Slack trace at all: the owning Situation's -recurrence count counts its already-closed predecessors and cannot move while -it is open, so `recurred ×N` shows only on the root of the next Situation the -group opens. A genuinely new incident whose key matches a past +branch the attach moves the owning Situation's recurrence count (closed +predecessors plus its own re-fires): the root shows `recurred ×N`, and a +crossed milestone rung is one quiet reply in the Situation's thread — never a +channel message. A genuinely new incident whose key matches a past analysis gets the prior finding **recalled** into its prompt as a past hypothesis, never as evidence. See [incident memory](incident-memory.md). diff --git a/docs/concepts/incident-memory.md b/docs/concepts/incident-memory.md index 46407c6..c0c4e78 100644 --- a/docs/concepts/incident-memory.md +++ b/docs/concepts/incident-memory.md @@ -24,11 +24,12 @@ When a firing alert's group key matches an already-analyzed incident and lands inside the **collapse horizon**, AlertINT attaches it as an **occurrence** of that incident instead of minting a new one and spending another analysis. In a released binary the incident's Slack card edits in place — `recurred ×N · -last HH:MM`. On the `state-controller` branch nothing is written to Slack at -all for that attach: the owning Situation's recurrence count counts the -Situations that closed before it and cannot move while it is open, so -`recurred ×N` shows only on the root of the next Situation the group opens. -Either way a JSON occurrence line is written to stdout. No second LLM call. +last HH:MM`. On the `state-controller` branch the attach moves the owning +Situation's recurrence count (its closed predecessors plus its own re-fires): +the root shows `recurred ×N`, and a crossed milestone rung records a +`recurrence_milestone` Transition with one quiet reply in the Situation's +thread. Either way a JSON occurrence line is written to stdout. No second LLM +call. The horizon is two clocks: a sliding attach window (default 30 minutes from the last occurrence) and a hard ceiling on the time since the last analysis (default diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 4f3374d..a8864f2 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -191,13 +191,13 @@ re-triaged as brand new every time it re-fires. When an alert whose group key matches an already-analyzed incident fires again inside the collapse horizon, it attaches as a lightweight occurrence instead of minting a new incident and spending another LLM call — a released binary edits the Incident card in place -to `recurred ×N`. On the `state-controller` branch an attach to a Situation -that is still live produces **no Slack trace at all**: a Situation's recurrence -count is the number of *already-closed* Situations for its group and is -therefore fixed for its whole lifetime, so `recurred ×N` appears only on the -root of the **next** Situation that opens for the group, rendered once at that -Situation's first publication. This is deterministic, free, and always on; -there is no enable switch, only the knobs below. +to `recurred ×N`. On the `state-controller` branch the attach feeds the owning +Situation's recurrence count (its closed predecessors plus its own re-fires); +the root shows `recurred ×N`, and crossing a milestone rung records a +`recurrence_milestone` Transition with one quiet thread reply (see +[Slack](../notifications/slack.md#recurrence-resurfacing)). This is +deterministic, free, and always on; there is no enable switch, only the knobs +below. | Field | Type | Default | Description | |---|---|---|---| @@ -378,7 +378,7 @@ starts when the aggregate LLM dependency state first becomes `degraded` or | `slack.bot_token_env` | string | — | Required when `slack.enabled: true`. Env var name holding the Slack bot token (`xoxb-…`, requires the `chat:write` scope; no history-read scope is ever requested) | | `slack.channel` | string | — | Required when `slack.enabled: true`. Channel name (e.g. `#alerts`) or ID (e.g. `C1234567890`) | | `slack.min_severity` | string | `low` | The channel-noise floor (`low` \| `medium` \| `high`); stdout always emits regardless. In a released binary it compares against the finding's severity, and an incident suppressed at firing is also suppressed at resolution. On the `state-controller` branch it is the minimum **interruption priority** a *new* main-channel interruption must meet — never alert severity and never a model claim; `critical` always passes, a withheld interruption is durably recorded, and the floor never suppresses Situation state, MCP history, a root edit, or a journal reply. The default posts everything. | -| `slack.recurrence_mode` | string | `change-gated` | How a recurring incident resurfaces in its thread: `change-gated` posts a thread reply only on a real-world change (severity rise, new symptom, faster cadence) or a milestone (×5/×10/×25/×50/×100, then every ×100) — replies stay in the thread, nothing extra is sent to the channel; `off` keeps recurrence to a silent card count-bump. **No effect on the `state-controller` branch**: an occurrence attaching to a live Situation posts nothing and edits nothing there, because that Situation's recurrence count cannot change while it is open — `recurred ×N` shows only on the root of the *next* Situation that opens for the group. The key is still accepted so an existing config keeps loading. See [Slack](../notifications/slack.md) for details. | +| `slack.recurrence_mode` | string | `change-gated` | How a recurring incident resurfaces in its thread: `change-gated` posts a thread reply only on a real-world change (severity rise, new symptom, faster cadence) or a milestone (×5/×10/×25/×50/×100, then every ×100) — replies stay in the thread, nothing extra is sent to the channel; `off` keeps recurrence to a silent card count-bump. On the `state-controller` branch the setting governs the owning Situation's milestone replies: `change-gated` posts one quiet reply in the Situation thread when the recurrence count crosses a rung, `off` keeps only the silent root edit; the `why:` change replies are released-binary only. See [Slack](../notifications/slack.md) for details. | At startup the agent logs one `notifiers ready` line listing the active sinks (and the Slack channel) so you can see where findings will go. Every analysis diff --git a/docs/notifications/slack.md b/docs/notifications/slack.md index bb290e2..c38d97e 100644 --- a/docs/notifications/slack.md +++ b/docs/notifications/slack.md @@ -418,14 +418,20 @@ notify: - `off` — recurrence never posts replies; the card's occurrence count still updates in place, silently. -On the integration branch this setting has **no effect**, and neither does a -re-fire that attaches to a Situation that is already open: it posts no reply -and edits no root. A Situation's recurrence count is the number of *closed* -Situations that preceded it in the same group, and only one Situation per -group can be open at a time — so that count is fixed for the Situation's whole -lifetime. `recurred ×N` therefore appears exactly once per Situation, on the -root of the **next** Situation the group opens, at its first publication. The -key is still accepted so an existing `config.yaml` keeps loading. +On the `state-controller` branch recurrence is owned by the Situation. A +Situation's recurrence count is the number of *closed* Situations that +preceded it in the same group **plus** every re-fire that attached to one of +its member incidents as an occurrence, so the count moves while the Situation +is open. The root renders it as `recurred ×N`. When the count crosses a +milestone rung (×5, ×10, ×25, ×50, ×100, then every ×100) the Situation +records a `recurrence_milestone` Transition, edits its root, and — under +`change-gated` — posts one quiet reply in its own thread; under `off` the +root edit still happens and no reply is posted. A milestone never re-pages +the channel, and a quiet Situation (one that never published) has no Slack +recurrence trace at all. The `why:` real-world-change replies above are a +released-binary feature; on this branch a real-world change reaches Slack as +the material Transition it is (a severity rise raises Attention, a new +symptom changes the assessment), not as a recurrence reply. ## System messages diff --git a/internal/situation/controller.go b/internal/situation/controller.go index c68fe7d..6fedaba 100644 --- a/internal/situation/controller.go +++ b/internal/situation/controller.go @@ -399,8 +399,24 @@ type ControllerConfig struct { // one (config situations.slack.repage_cooldown_seconds). Default 900s; // it gates exactly one poke class (PokeRequiredActionChanged). RepageCooldown time.Duration + + // RecurrenceMode is the operator's notify.slack.recurrence_mode, kept + // from the legacy presentation path (spec.md "Journal": "Plan 3 + // preserves current recurrence configuration and milestones"): + // RecurrenceModeChangeGated (the default, also for the empty value) + // posts a recurrence milestone as a quiet thread entry in the owning + // Situation thread; RecurrenceModeOff never posts one — the milestone + // Transition still exists and still edits the root's count. Neither + // ever re-pages the channel. + RecurrenceMode string } +// Recurrence modes — the accepted values of notify.slack.recurrence_mode. +const ( + RecurrenceModeChangeGated = "change-gated" + RecurrenceModeOff = "off" +) + const ( defaultControllerMaxL2CallsPerAttempt = 2 defaultControllerMaxWorkAttemptsPerInput = 5 @@ -1142,6 +1158,7 @@ func (c *Controller) buildHistory(claim Claim, basis historyBasis, commit Contro LastMainChannelPokeAt: basis.In.LastMainChannelPokeAt, SlackFloor: c.cfg.SlackFloor, RepageCooldown: c.cfg.RepageCooldown, + RecurrenceRepliesOff: c.cfg.RecurrenceMode == RecurrenceModeOff, Drill: change.Drill, Now: basis.Now, } @@ -1161,6 +1178,17 @@ func (c *Controller) buildHistory(claim Claim, basis historyBasis, commit Contro return &history, nil } +// recurrenceCountOf is the Situation's durable recurrence count: prior +// terminal Situations in its exact group plus its member Incidents' +// recurrence-collapse occurrences. +func recurrenceCountOf(in SnapshotInput) int { + count := len(in.PriorSituations) + for _, inc := range in.Incidents { + count += inc.Occurrences + } + return count +} + // authoritativeChangeOf reduces one committed reconciliation to exactly what // deriving durable history needs. The Situation it carries is the COMMITTED // projection — Plan 2's own lifecycle/Attention/recovery/terminal decisions @@ -1207,10 +1235,14 @@ func authoritativeChangeOf(claim Claim, basis historyBasis, commit ControllerCom Incidents: basis.Snap.Incidents, TriageDecisions: commit.TriageDecisions, // spec.md: "recurrence count available from durable local Store - // facts" — this exact group's prior terminal Situations, the same + // facts" — this exact group's prior terminal Situations (the same // durable lineage Plan 2 already loads for its duration - // distribution. No new counting machinery. - RecurrenceCount: len(basis.In.PriorSituations), + // distribution) plus every re-fire that attached to a member + // Incident as a recurrence-collapse occurrence. The prior count + // alone is fixed for a Situation's whole lifetime (one nonterminal + // Situation per group), so occurrences are what let a milestone + // actually be reached while it is open (review round 1, R1-F6). + RecurrenceCount: recurrenceCountOf(basis.In), OperatorArtifacts: basis.In.PendingArtifacts, Drill: situationDrill(basis.In), Now: basis.Now, diff --git a/internal/situation/controller_test.go b/internal/situation/controller_test.go index b2d6f02..7af00bc 100644 --- a/internal/situation/controller_test.go +++ b/internal/situation/controller_test.go @@ -1896,6 +1896,26 @@ func TestControllerHistoryRecurrenceCountComesFromPriorTerminalSituations(t *tes } } +// TestControllerHistoryRecurrenceCountAddsMemberOccurrences pins the other +// half of the durable recurrence count: every re-fire that attached to a +// member Incident as a recurrence-collapse occurrence, which is what lets a +// milestone be reached while the Situation is open (review round 1, R1-F6). +func TestControllerHistoryRecurrenceCountAddsMemberOccurrences(t *testing.T) { + in := ctReuseInput(t) + in.PriorSituations = []situation.CompletedSituation{ + {ID: "prior-a", GroupKey: "group-1", EffectiveStartedAt: ctBaseTime.Add(-48 * time.Hour), TerminalAt: ctBaseTime.Add(-47 * time.Hour), TerminalReason: model.TerminalReasonObservationDeadline}, + } + in.Incidents[0].Occurrences = 4 + + commit := ctReconcileOnce(t, in, ctBaseClaim(), nil) + if commit.History == nil || commit.History.Summary == nil { + t.Fatalf("expected history, got %+v", commit.History) + } + if got := commit.History.Summary.RecurrenceCount; got != 5 { + t.Fatalf("summary recurrence count = %d, want 5 (one prior Situation plus four occurrences)", got) + } +} + // TestControllerHistoryBlockedCycleStillCommitsHistory covers the blocked // result class: a cycle that may not dispatch further L2 work still // establishes authoritative state, so it still records the history that diff --git a/internal/situation/history_replay_test.go b/internal/situation/history_replay_test.go index caa1313..ab838cc 100644 --- a/internal/situation/history_replay_test.go +++ b/internal/situation/history_replay_test.go @@ -1509,16 +1509,16 @@ func scenarioRecurrenceLineageAndHandoff() historyScenario { // assertRecurrenceMilestoneReached proves the live Situation's durable // recurrence count reached want, and that its Episode summary carries it. // -// Note deliberately NOT asserted here: a `recurrence_milestone` Transition -// REASON. RecurrenceCount is len(prior terminal Situations for this exact -// group), and migration 0014's situations_one_nonterminal_group_idx allows -// at most one nonterminal Situation per group — so no sibling can -// terminalize while this Situation is live, and the count (hence the -// milestone rung in the materiality tuple) is fixed for its whole lifetime. -// The reason itself is therefore unreachable from the natural pipeline in -// this build; history_test.go's own TestBuildTransitionsCatalog covers it -// at the derivation level. What replay must prove here is that the durable -// recurrence count and its milestone rung survive a crash unchanged. +// The count is prior terminal Situations for this exact group plus member +// Incidents' recurrence-collapse occurrences; this scenario drives only the +// lineage half, so the rung is reached at the first Transition rather than +// by a later `recurrence_milestone` Transition. That later path — a re-fire +// attaching as an occurrence, the membership input, the milestone +// Transition and its quiet thread entry — is driven end to end by +// cmd/alertint's fake-Slack test +// (TestSituationSlackE2ERecurrenceMilestoneStaysInThread). What replay +// must prove here is that the durable recurrence count and its milestone +// rung survive a crash unchanged. func assertRecurrenceMilestoneReached(t *testing.T, st *store.Store, want int) { t.Helper() var got int diff --git a/internal/situation/notification_plan.go b/internal/situation/notification_plan.go index a6dfba6..6eecf32 100644 --- a/internal/situation/notification_plan.go +++ b/internal/situation/notification_plan.go @@ -46,8 +46,12 @@ type PublicationInput struct { LastMainChannelPokeAt *time.Time SlackFloor model.InterruptionPriority RepageCooldown time.Duration - Drill bool - Now time.Time + // RecurrenceRepliesOff is notify.slack.recurrence_mode = off: a + // recurrence milestone still creates its Transition and still edits + // the root (the count updates in place), but posts no thread entry. + RecurrenceRepliesOff bool + Drill bool + Now time.Time } // PlanNotificationIntents derives every durable Slack obligation one @@ -151,6 +155,12 @@ func PlanNotificationIntents(in PublicationInput) ([]model.NotificationIntent, e if tr.JournalKind == model.JournalNone && !poked { continue } + if tr.Reason == model.ReasonRecurrenceMilestone && in.RecurrenceRepliesOff { + // recurrence_mode: off keeps recurrence to the root's silent + // count update (the root_sync above); a milestone is never a + // poke, so nothing else is lost. + continue + } if !poked { out = append(out, newIntent(in, model.EffectThreadAppend, tr, threadKey(model.EffectThreadAppend, in.Situation.ID, tr.Sequence))) diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go index 4140ff0..eb01e05 100644 --- a/internal/situation/notification_plan_test.go +++ b/internal/situation/notification_plan_test.go @@ -1053,3 +1053,49 @@ func TestPublicationAuthorityComesFromFloorOrValidatedReason(t *testing.T) { t.Error("urgent Attention is only reachable through a deterministic floor and is publication authority") } } + +// ---------------------------------------------------------------------- +// Recurrence mode (review round 1, R1-F6). +// ---------------------------------------------------------------------- + +// hsMilestoneCommit builds a published Situation's recurrence-milestone +// commit: the recurrence count crosses the first rung with nothing else +// changing. +func hsMilestoneCommit(t *testing.T) (AuthoritativeChange, []model.Transition, model.EpisodeSummary) { + t.Helper() + c := hsNext(t) + c.RecurrenceCount = 5 + trs, sum := hsCommitOf(t, c) + if len(trs) != 1 || trs[0].Reason != model.ReasonRecurrenceMilestone { + t.Fatalf("fixture: want one recurrence_milestone transition, got %+v", trs) + } + return c, trs, sum +} + +func TestPlanNotificationIntentsRecurrenceModeOffKeepsTheRootEditOnly(t *testing.T) { + c, trs, sum := hsMilestoneCommit(t) + in := hsPub(c, trs, sum) + in.RecurrenceRepliesOff = true + + got := hsPlan(t, in) + if roots := hsIntentsOfClass(got, model.EffectRootSync); len(roots) != 1 || roots[0].Status != model.IntentPending { + t.Fatalf("recurrence_mode off: want the silent root edit (the count updates in place), got %+v", roots) + } + if replies := hsReplyIntents(got); len(replies) != 0 { + t.Fatalf("recurrence_mode off: a milestone posted %d thread entries, want none", len(replies)) + } +} + +func TestPlanNotificationIntentsRecurrenceModeChangeGatedPostsAQuietMilestone(t *testing.T) { + c, trs, sum := hsMilestoneCommit(t) + in := hsPub(c, trs, sum) // RecurrenceRepliesOff false: change-gated + + got := hsPlan(t, in) + threads := hsIntentsOfClass(got, model.EffectThreadAppend) + if len(threads) != 1 || threads[0].Status != model.IntentPending { + t.Fatalf("change-gated: want one quiet milestone thread entry, got %+v", threads) + } + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { + t.Fatalf("a recurrence milestone must never re-page the channel, got %+v", broadcasts) + } +} diff --git a/internal/situation/snapshot.go b/internal/situation/snapshot.go index 13f76f2..5f7714d 100644 --- a/internal/situation/snapshot.go +++ b/internal/situation/snapshot.go @@ -215,6 +215,16 @@ type IncidentState struct { ReadyAt time.Time AlertCount int Triage TriageState + + // Occurrences is how many times this Incident's condition re-fired and + // attached as a recurrence-collapse occurrence (incident_occurrences + // rows) — the durable local Store fact behind the Situation's + // recurrence count and its milestones (spec.md: "recurrence count + // available from durable local Store facts"; review round 1, R1-F6). + // It is not part of any digest or hash: a re-fire reaches the + // controller as its own Situation input, and only the milestone RUNG + // is material. + Occurrences int } // CompletedSituation is one prior terminal Situation in the same exact-group diff --git a/internal/store/situation_controller.go b/internal/store/situation_controller.go index 8b0e049..07eb30b 100644 --- a/internal/store/situation_controller.go +++ b/internal/store/situation_controller.go @@ -423,14 +423,17 @@ func loadSituationDeliveriesTx(ctx context.Context, tx *sql.Tx, situationID stri // loadSituationIncidentStatesTx reads every current member Incident of // situationID plus its current incident_triage row (LEFT JOIN: an Incident -// that has never reached "ready" has none — TriageState.Phase stays ""). +// that has never reached "ready" has none — TriageState.Phase stays "") and +// its recurrence-collapse occurrence count (incident_occurrences), the +// durable fact behind the Situation's recurrence milestones. func loadSituationIncidentStatesTx(ctx context.Context, tx *sql.Tx, situationID string) ([]situation.IncidentState, error) { rows, err := tx.QueryContext(ctx, ` SELECT i.id, i.group_key, i.status, i.first_alert_at, i.last_alert_at, i.ready_at, i.alert_count, COALESCE(t.phase, ''), COALESCE(t.attempts, 0), t.next_at, t.decision, t.decision_reason, t.decision_input_version, t.material_fact_hash, t.membership_digest, t.incident_input_digest, - t.assessment_id, t.decided_at + t.assessment_id, t.decided_at, + (SELECT COUNT(*) FROM incident_occurrences o WHERE o.incident_id = i.id) FROM situation_incidents si JOIN incidents i ON i.id = si.incident_id LEFT JOIN incident_triage t ON t.incident_id = i.id @@ -451,7 +454,7 @@ func loadSituationIncidentStatesTx(ctx context.Context, tx *sql.Tx, situationID &st.Triage.Phase, &st.Triage.Attempts, &nextAt, &decision, &decisionReason, &decisionInputVersion, &materialHash, &membershipDigest, &incidentInputDigest, - &assessmentID, &decidedAt); err != nil { + &assessmentID, &decidedAt, &st.Occurrences); err != nil { return nil, fmt.Errorf("store: scan situation incident state: %w", err) } diff --git a/internal/store/situation_controller_test.go b/internal/store/situation_controller_test.go index 60ce483..21d8101 100644 --- a/internal/store/situation_controller_test.go +++ b/internal/store/situation_controller_test.go @@ -1206,6 +1206,9 @@ func TestLoadReconciliationInputReadsCoherentSnapshot(t *testing.T) { if len(snap.Incidents) != 1 || snap.Incidents[0].ID != incID { t.Fatalf("snapshot incidents = %+v, want exactly the one member incident", snap.Incidents) } + if snap.Incidents[0].Occurrences != 0 { + t.Fatalf("occurrences = %d before any re-fire, want 0", snap.Incidents[0].Occurrences) + } if snap.Incidents[0].Triage.Phase != "awaiting_decision" { t.Fatalf("triage phase = %q, want awaiting_decision", snap.Incidents[0].Triage.Phase) } @@ -2493,3 +2496,36 @@ func TestCommitControllerPersistsEligibleReasonCandidateSet(t *testing.T) { t.Fatalf("eligible reasons after an empty commit = %#v, want a non-nil empty slice", view.EligibleReasons) } } + +// TestLoadReconciliationInputCountsMemberOccurrences proves a member +// Incident's recurrence-collapse occurrences reach the snapshot — the +// durable fact behind the Situation's recurrence milestones (review round +// 1, R1-F6). +func TestLoadReconciliationInputCountsMemberOccurrences(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + sitID, _ := snSeedOneCycle(t, st, "group-occurrences", now) + + var incID string + if err := st.db.QueryRowContext(ctx, `SELECT incident_id FROM situation_incidents WHERE situation_id = ?`, sitID).Scan(&incID); err != nil { + t.Fatalf("member incident: %v", err) + } + for i := 0; i < 3; i++ { + if _, err := st.InsertOccurrence(ctx, Occurrence{ + IncidentID: incID, OccurredAt: now.Add(time.Duration(i+1) * time.Minute), + Fingerprints: []string{"fp-occ"}, Payload: []OccurrenceMember{}, TriggerKind: "none", + }); err != nil { + t.Fatalf("InsertOccurrence %d: %v", i, err) + } + } + shMakeDue(t, st, sitID, now.Add(-time.Minute)) + claim := claimSituation(t, st, sitID, "controller-occ", now.Add(5*time.Minute)) + in, err := st.LoadReconciliationInput(ctx, claim, now.Add(5*time.Minute)) + if err != nil { + t.Fatalf("LoadReconciliationInput: %v", err) + } + if len(in.Incidents) != 1 || in.Incidents[0].Occurrences != 3 { + t.Fatalf("member occurrences = %+v, want exactly 3 on the one member", in.Incidents) + } +} From 486249d4a51207ae4e769375678731f6a5e79fc5 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 14:03:36 +0300 Subject: [PATCH 26/31] test(store): satisfy rowserrcheck, sqlclosecheck, and dogsled in review tests Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- ...ation_supersede_live_roots_upgrade_test.go | 38 ++++++++++++------- .../store/situation_notifications_test.go | 8 ++-- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/internal/store/notification_supersede_live_roots_upgrade_test.go b/internal/store/notification_supersede_live_roots_upgrade_test.go index 936c824..4a224fe 100644 --- a/internal/store/notification_supersede_live_roots_upgrade_test.go +++ b/internal/store/notification_supersede_live_roots_upgrade_test.go @@ -128,19 +128,7 @@ func TestNotificationSupersedeLiveRootsUpgrade(t *testing.T) { if err := st.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pragma_foreign_key_check`).Scan(&fkViolations); err != nil || fkViolations != 0 { t.Fatalf("foreign_key_check violations = %d (err=%v), want 0", fkViolations, err) } - var triggers string - rows, err := st.db.QueryContext(ctx, `SELECT name FROM sqlite_master WHERE type = 'trigger' AND name LIKE 'notification_intents_supersede%'`) - if err != nil { - t.Fatalf("list triggers: %v", err) - } - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - t.Fatalf("scan trigger: %v", err) - } - triggers += name + ";" - } - _ = rows.Close() + triggers := supersessionTriggerNames(t, st) if strings.Contains(triggers, "from_pending_only") || !strings.Contains(triggers, "from_live_only") { t.Fatalf("supersession triggers after upgrade = %q, want only notification_intents_supersede_from_live_only", triggers) } @@ -163,3 +151,27 @@ func TestNotificationSupersedeLiveRootsUpgrade(t *testing.T) { t.Fatalf("superseding a delivered root = %v, want the 0020 trigger's rejection", err) } } + +// supersessionTriggerNames lists the supersession triggers currently +// defined on notification_intents, ";"-joined. +func supersessionTriggerNames(t *testing.T, st *Store) string { + t.Helper() + rows, err := st.db.QueryContext(context.Background(), + `SELECT name FROM sqlite_master WHERE type = 'trigger' AND name LIKE 'notification_intents_supersede%'`) + if err != nil { + t.Fatalf("list triggers: %v", err) + } + defer func() { _ = rows.Close() }() + var triggers string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan trigger: %v", err) + } + triggers += name + ";" + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate triggers: %v", err) + } + return triggers +} diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index 9f9895a..e08e32a 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -1073,9 +1073,8 @@ func TestMarkNotificationDeliveredSupersededFirstPostKeepsCoordinates(t *testing t.Fatalf("claimed %s, want the replacement root projection", replacement.Intent.EffectClass) } snDeliver(t, st, replacement, "100.1", now.Add(4*time.Second)) - _, ts, _, _ = st.GetSituationRootCoordinates(ctx, id) - if ts != "100.1" { - t.Fatalf("root coordinates = %q after the replacement delivered, want the first post's 100.1", ts) + if _, ts, _, err := st.GetSituationRootCoordinates(ctx, id); err != nil || ts != "100.1" { + t.Fatalf("root coordinates = %q (err=%v) after the replacement delivered, want the first post's 100.1", ts, err) } } @@ -1242,5 +1241,8 @@ func snDump(t *testing.T, st *Store) string { fmt.Fprintf(&out, "\n %s class=%s status=%s seq=%d retry_at=%q owner=%q err=%q requires_root=%d", id[:8], class, status, seq, retryAt, owner, errClass, requiresRoot) } + if err := rows.Err(); err != nil { + t.Fatalf("iterate intents: %v", err) + } return out.String() } From e2c2bd3907d35ac7591acf68909676414c22b639 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 14:47:24 +0300 Subject: [PATCH 27/31] fix(situation): revalidate a handoff on the requested human action only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HandoffStillCurrent compared the whole Operator contract (minus its deadline). That tuple includes AlertINT's action, status, wait reason, and update triggers, which move as Triage progresses or completes while the required human action is exactly as outstanding — so a queued handoff was demoted to a delayed "no longer current" thread entry when AlertINT's own work changed (review round 2, R2-F2). The revalidation now compares the requested operator action on its own; terminal recovery, Attention de-escalation, and a withdrawn or changed human action still demote. PokeRequiredActionChanged keeps its contract basis (the operator-action catalog has one entry in this build, so a narrower class would be unreachable); the asymmetry and its interaction with the repage cooldown are documented on HandoffStillCurrent. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- internal/situation/priority.go | 25 ++++++++++++---- internal/situation/priority_test.go | 46 +++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/internal/situation/priority.go b/internal/situation/priority.go index fb8d964..11570c8 100644 --- a/internal/situation/priority.go +++ b/internal/situation/priority.go @@ -177,11 +177,24 @@ func ClassifyPoke(prior *model.Transition, t model.Transition) PokeClass { // - a terminal Situation has no current interruption; // - de-escalated Attention demotes the poke it followed; // - a handoff that asked the operator for something stays current only -// while the current Operator contract still asks for the same thing -// (operatorContractTuple, the same basis PokeRequiredActionChanged is -// judged on); an escalation poke that asked for no operator action -// (newly urgent Attention, newly crossed criticality) is current while -// its Attention still holds. +// while the current Operator contract still asks the operator for the +// SAME HUMAN ACTION — compared on its own, never on the rest of the +// contract: AlertINT's action/status, wait reason, and update triggers +// move as Triage progresses or completes while the outstanding human +// action is exactly as outstanding (review round 2, R2-F2); an +// escalation poke that asked for no operator action (newly urgent +// Attention, newly crossed criticality) is current while its Attention +// still holds. +// +// This is deliberately narrower than PokeRequiredActionChanged's basis +// (the whole contract without its deadline): that class decides whether a +// NEW interruption is warranted, this function decides whether an +// interruption already owed may be demoted. With the current one-entry +// operator-action catalog the two can disagree on an internal contract +// change — the queued handoff still broadcasts as current, and the newer +// Transition may broadcast again once the repage cooldown allows — which is +// the side to err on: the spec demotes only when the requested action is +// no longer current. // // A summary that does not yet include the handoff cannot confirm it and // counts as not current. @@ -196,5 +209,5 @@ func HandoffStillCurrent(handoff model.Transition, summary model.EpisodeSummary) return true } return summary.ActionContract.OperatorActionRequired != nil && - operatorContractTuple(summary.ActionContract) == operatorContractTuple(handoff.ActionContract) + derefOperatorAction(summary.ActionContract.OperatorActionRequired) == derefOperatorAction(handoff.ActionContract.OperatorActionRequired) } diff --git a/internal/situation/priority_test.go b/internal/situation/priority_test.go index 2d7fe59..f9b6435 100644 --- a/internal/situation/priority_test.go +++ b/internal/situation/priority_test.go @@ -401,3 +401,49 @@ func TestHandoffStillCurrentEscalationPokeNeedsNoOperatorAction(t *testing.T) { t.Error("an escalation poke is demoted once Attention de-escalates") } } + +// TestHandoffStillCurrentIgnoresAlertINTMachinery pins review round 2, +// R2-F2: contracts derived through the production DeriveActionContract for +// "Triage in flight" and "Triage complete" differ in AlertINT's action, +// status, and update triggers while the required human action is +// identical — the queued handoff is still current. +func TestHandoffStillCurrentIgnoresAlertINTMachinery(t *testing.T) { + c := hsNext(t) + action := model.OperatorActionInvestigateSituation + state := ControllerState{Lifecycle: model.LifecycleActive, Attention: model.AttentionInvestigate, + OperatorActionRequired: &action, TriagePhase: TriagePhaseInFlight} + handoff := *c.PriorTransition + handoff.ActionContract = DeriveActionContract(state, DeriveCadence(state), c.Now) + summary := *c.PriorSummary + summary.SourceTransitionSequence = handoff.Sequence + 1 + + // Triage completed; nothing is in flight; the human action is still owed. + state.TriagePhase = TriagePhaseNone + summary.ActionContract = DeriveActionContract(state, DeriveCadence(state), c.Now) + if err := summary.Validate(); err != nil { + t.Fatal(err) + } + if operatorContractTuple(summary.ActionContract) == operatorContractTuple(handoff.ActionContract) { + t.Fatal("fixture: the two contracts must differ in AlertINT machinery") + } + if !HandoffStillCurrent(handoff, summary) { + t.Fatal("AlertINT work completed, but the identical outstanding operator action was treated as no longer current") + } + + // Different reconsideration triggers, same human action: still current. + summary.ActionContract = handoff.ActionContract + summary.ActionContract.NextUpdateOn = []model.NextUpdateOn{model.NextUpdateOnSourceResolution} + if !HandoffStillCurrent(handoff, summary) { + t.Fatal("a changed update trigger is AlertINT machinery, not a changed human action") + } + + // The human action withdrawn: demoted. + summary.ActionContract = DeriveActionContract(ControllerState{Lifecycle: model.LifecycleActive, + Attention: model.AttentionInvestigate, TriagePhase: TriagePhaseNone}, model.CadenceNormal, c.Now) + if summary.ActionContract.OperatorActionRequired != nil { + t.Fatal("fixture: the withdrawn contract must carry no operator action") + } + if HandoffStillCurrent(handoff, summary) { + t.Fatal("a withdrawn human action must demote the handoff") + } +} From 043dcfa795c787504cc9a6079c256ae01a169db7 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 14:47:24 +0300 Subject: [PATCH 28/31] fix(store): count only real obligations in gap replay; coalesce old roots on reactivation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R2-F1: both gap-accounting queries treated every pending Situation intent as replayable. A reply planned under an unpublished, floor-withheld root can never deliver and was never delayed by the outage, yet it inflated the recovery notice's affected count and kept CompleteDeliveryGap false forever. replayableIntentPredicate now defines the backlog as an actual delivery obligation: a root projection always, a reply only while its Situation has a published root or a live root projection still owed; the same predicate drives the notice counts and completion, and the journal counts again as soon as a later commit earns the Situation a root. R2-F4: reactivateBlockedRootSyncTx returned early when the newest live root was already pending, before retiring an older blocked one. A schema-19 ledger can hold exactly that pair (its trigger forbade retiring the blocked row), and with blocked rows holding the claim queue the old row stalled the Situation forever — a terminal one gets no controller commit to repair it. Reactivation now picks the newest root of any delivered or live status, coalesces every other live projection into it first, then reactivates it only if it is the blocked one; an old blocked root behind a delivered newer one is retired rather than revived. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- internal/store/notification_gaps.go | 94 ++++++++---- internal/store/notification_gaps_test.go | 135 ++++++++++++++++++ ...ation_supersede_live_roots_upgrade_test.go | 56 +++++++- 3 files changed, 252 insertions(+), 33 deletions(-) diff --git a/internal/store/notification_gaps.go b/internal/store/notification_gaps.go index 3f2ffab..a0221cd 100644 --- a/internal/store/notification_gaps.go +++ b/internal/store/notification_gaps.go @@ -38,10 +38,12 @@ import ( // // Which intents a generation is replaying is DERIVED, not stored: migration // 0018 deliberately allows gap_generation only on the recovery notice -// itself, so "replayable" means "a pending Situation-scoped intent that -// already existed when this generation recovered" (created_at <= -// recovered_at). That keeps work committed after recovery — ordinary -// delivery, not replay — from holding a generation open forever. +// itself, so "replayable" means "a pending Situation-scoped intent that is +// an actual delivery obligation and already existed when this generation +// recovered" (replayableIntentPredicate, created_at <= recovered_at). That +// keeps work committed after recovery — ordinary delivery, not replay — +// and history that has no root to deliver under from holding a generation +// open forever. // ---------------------------------------------------------------------- // GapSnapshot is one durable gap generation's bounded rendering facts: the @@ -357,8 +359,8 @@ func (s *Store) CompleteDeliveryGap(ctx context.Context, now time.Time) (string, var remaining int if err := tx.QueryRowContext(ctx, ` - SELECT COUNT(*) FROM notification_intents - WHERE status = 'pending' AND situation_id IS NOT NULL AND created_at <= ?`, recoveredAt). + SELECT COUNT(*) FROM notification_intents ni + WHERE `+replayableIntentPredicate, recoveredAt). Scan(&remaining); err != nil { return "", false, fmt.Errorf("store: count replayable notification intents: %w", err) } @@ -438,15 +440,33 @@ func (s *Store) ReactivateConfigurationBlocked(ctx context.Context, configuratio return n, nil } +// replayableIntentPredicate selects the pending Situation-scoped intents +// that are an ACTUAL delivery obligation as of the bound instant (the one +// `?`): a root projection always is; a reply is only while its Situation +// has a published root or a live (pending, configuration-blocked, or +// failed) root projection still owed to Slack. A reply planned under an +// unpublished, floor-withheld root has no root to hang under and none +// coming — it is immutable local history, not delayed Slack work — so it +// neither counts toward a recovery notice nor holds a generation in replay +// (review round 2, R2-F1). It becomes an obligation again the moment a +// later commit earns the Situation a root. +const replayableIntentPredicate = `ni.status = 'pending' AND ni.situation_id IS NOT NULL AND ni.created_at <= ? + AND (ni.requires_root = 0 + OR EXISTS (SELECT 1 FROM situations s WHERE s.id = ni.situation_id AND s.slack_root_ts IS NOT NULL) + OR EXISTS (SELECT 1 FROM notification_intents r + WHERE r.situation_id = ni.situation_id AND r.effect_class = 'root_sync' + AND r.status IN ('pending', 'blocked_configuration', 'failed')))` + // replayableBacklogTx counts the Situation-scoped delivery obligations -// outstanding as of asOf: how many distinct Situations, and how many -// effects. These are exactly the numbers the recovery notice reports. +// outstanding as of asOf (replayableIntentPredicate): how many distinct +// Situations, and how many effects. These are exactly the numbers the +// recovery notice reports. func replayableBacklogTx(ctx context.Context, tx *sql.Tx, asOf string) (int, int, error) { var affected, delayed int if err := tx.QueryRowContext(ctx, ` - SELECT COUNT(DISTINCT situation_id), COUNT(*) - FROM notification_intents - WHERE status = 'pending' AND situation_id IS NOT NULL AND created_at <= ?`, asOf). + SELECT COUNT(DISTINCT ni.situation_id), COUNT(*) + FROM notification_intents ni + WHERE `+replayableIntentPredicate, asOf). Scan(&affected, &delayed); err != nil { return 0, 0, fmt.Errorf("store: count delayed notification backlog: %w", err) } @@ -507,39 +527,55 @@ type liveRootProjection struct { status string } -// reactivateBlockedRootSyncTx restores exactly one pending root projection -// for situationID and reports how many blocked roots it reactivated (0 or 1). +// reactivateBlockedRootSyncTx normalizes situationID's root projections on +// corrected configuration and reports how many blocked roots it +// reactivated (0 or 1). +// +// The keeper is the NEWEST root projection of any delivered or live status +// (by summary version, then creation) — the one that renders the most +// current state. Every OTHER live projection (pending, blocked, failed) is +// coalesced into it first, whatever the keeper's own status: a newer +// commit does exactly this at commit time (supersedeLiveRootSyncTx), but a +// ledger written under migration 0018's "supersede from pending only" +// trigger can still hold an older blocked root beside a newer pending or +// delivered one, and with blocked rows holding the claim queue such a row +// would stall the Situation forever — a terminal Situation gets no +// controller commit to repair it (review round 2, R2-F4). Then: // -// A newer root projection supersedes every older live one at commit -// (supersedeLiveRootSyncTx), so a Situation normally holds ONE live root: -// if it is pending, corrected configuration has nothing to do here — that -// projection delivers the coordinates every dependent effect waits on; if -// it is blocked, it becomes pending. Any older live projection that somehow -// survived is coalesced into the newest one first, so migration 0018's -// single-pending-root index is satisfied at every step. +// - keeper pending: nothing more to do; it delivers the coordinates every +// dependent effect waits on; +// - keeper delivered: nothing more to do; the Situation's root is on +// screen and an older blocked projection was obsolete; +// - keeper blocked: it becomes pending; +// - keeper failed: left for an explicit redrive (spec.md: failed is never +// auto-retried). +// +// Migration 0018's single-pending-root index is satisfied at every step: +// the coalescing retires every other pending row before the keeper is +// made pending. func reactivateBlockedRootSyncTx(ctx context.Context, tx *sql.Tx, situationID, nowStr string) (int, error) { rows, err := tx.QueryContext(ctx, ` SELECT id, status FROM notification_intents WHERE situation_id = ? AND effect_class = 'root_sync' - AND status IN ('pending','blocked_configuration') - ORDER BY created_at ASC, id ASC`, situationID) + AND status IN ('pending', 'blocked_configuration', 'failed', 'delivered') + ORDER BY summary_version ASC, created_at ASC, id ASC`, situationID) if err != nil { - return 0, fmt.Errorf("store: list live root projections for %s: %w", situationID, err) + return 0, fmt.Errorf("store: list root projections for %s: %w", situationID, err) } - live, err := scanLiveRootProjections(rows) + roots, err := scanLiveRootProjections(rows) if err != nil { return 0, err } - if len(live) == 0 { - return 0, nil - } - keeper := live[len(live)-1] - if keeper.status == string(situationmodel.IntentPending) { + if len(roots) == 0 { return 0, nil } + keeper := roots[len(roots)-1] if err := supersedeLiveRootSyncTx(ctx, tx, situationID, keeper.id); err != nil { return 0, err } + if keeper.status != string(situationmodel.IntentBlockedConfiguration) { + return 0, nil + } if err := setNotificationIntentPendingTx(ctx, tx, keeper.id, "blocked_configuration", nowStr); err != nil { return 0, err } diff --git a/internal/store/notification_gaps_test.go b/internal/store/notification_gaps_test.go index 494476b..891ab15 100644 --- a/internal/store/notification_gaps_test.go +++ b/internal/store/notification_gaps_test.go @@ -395,3 +395,138 @@ func TestDeliveryGapConfigurationGenerationReactivatesBlockedIntents(t *testing. t.Fatalf("reactivated intent = %+v, want pending with its attempts preserved", reactivated) } } + +// TestDeliveryGapWithheldEpisodeDoesNotHoldReplay pins review round 2, +// R2-F1: a floor-withheld, never-published Situation's pending journal is +// not delayed Slack work — it neither counts toward the recovery notice +// nor keeps a generation replaying once every actual obligation drained. +func TestDeliveryGapWithheldEpisodeDoesNotHoldReplay(t *testing.T) { + st := newTestStore(t) + ctx := context.Background() + now := time.Date(2026, 9, 6, 10, 0, 0, 0, time.UTC) + withheldID := snSeedWithheldEpisode(t, st, "review-withheld", now) + snSeedOneCycle(t, st, "review-deliverable", now) + + snFail(t, st, now) + if _, err := st.OpenDueDeliveryGap(ctx, now.Add(5*time.Minute), 5*time.Minute); err != nil { + t.Fatal(err) + } + generation, _, err := st.RecoverDeliveryGap(ctx, now.Add(7*time.Minute)) + if err != nil { + t.Fatal(err) + } + gap, err := st.GetDeliveryGap(ctx, generation) + if err != nil { + t.Fatal(err) + } + if gap.AffectedSituationCount != 1 { + t.Errorf("recovery notice counts %d affected Situations, want 1: the floor-withheld episode had no Slack delivery obligation", gap.AffectedSituationCount) + } + for round := 0; round < 8; round++ { + at := now.Add(8*time.Minute + time.Duration(round)*time.Second) + claims, err := st.ClaimNotificationIntents(ctx, snOwner, at, time.Minute, 25) + if err != nil { + t.Fatal(err) + } + for _, c := range claims { + snDeliver(t, st, c, "100.1", at) + } + } + if _, complete, err := st.CompleteDeliveryGap(ctx, now.Add(9*time.Minute)); err != nil || !complete { + t.Fatalf("CompleteDeliveryGap = (%v, %v): all deliverable backlog drained, but the withheld root's pending journal keeps the gap replaying", complete, err) + } + + // Later publication authority makes that journal an obligation again: + // a second outage's generation counts it, and completes only once it + // (and its root) delivered. + // The worker records the replay's delivery successes; that closes the + // first outage's window before the second outage begins. + if err := st.ObserveSlackSuccess(ctx, now.Add(9*time.Minute)); err != nil { + t.Fatal(err) + } + snCommitAboveFloor(t, st, withheldID, now.Add(20*time.Minute)) + snFail(t, st, now.Add(21*time.Minute)) + if opened, err := st.OpenDueDeliveryGap(ctx, now.Add(26*time.Minute), 5*time.Minute); err != nil || !opened { + t.Fatalf("second generation = (%v, %v), want opened", opened, err) + } + second, _, err := st.RecoverDeliveryGap(ctx, now.Add(27*time.Minute)) + if err != nil { + t.Fatal(err) + } + gap, err = st.GetDeliveryGap(ctx, second) + if err != nil { + t.Fatal(err) + } + if gap.AffectedSituationCount != 1 || gap.DelayedEffectCount < 2 { + t.Fatalf("second notice = %d affected / %d delayed, want the now-warranted Situation with its root and journal", gap.AffectedSituationCount, gap.DelayedEffectCount) + } + if _, complete, err := st.CompleteDeliveryGap(ctx, now.Add(28*time.Minute)); err != nil || complete { + t.Fatalf("second generation completed before its backlog delivered (complete=%v err=%v)", complete, err) + } + for round := 0; round < 8; round++ { + at := now.Add(28*time.Minute + time.Duration(round)*time.Second) + claims, err := st.ClaimNotificationIntents(ctx, snOwner, at, time.Minute, 25) + if err != nil { + t.Fatal(err) + } + for _, c := range claims { + snDeliver(t, st, c, "200.1", at) + } + } + if _, complete, err := st.CompleteDeliveryGap(ctx, now.Add(29*time.Minute)); err != nil || !complete { + t.Fatalf("second generation did not complete after its backlog delivered (complete=%v err=%v)", complete, err) + } +} + +// snSeedWithheldEpisode commits one warranted-but-below-floor first cycle +// for group: a withheld root plus a pending requires_root journal entry. +func snSeedWithheldEpisode(t *testing.T, st *Store, group string, now time.Time) string { + t.Helper() + ctx := context.Background() + id := newSituationForGroup(t, st, group, now) + claim := claimSituation(t, st, id, "controller-a", now) + cycle := shPrepare(t, claim, shRunningTriageContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + conclusion := *cycle.Change.Projection.Assessment + conclusion.SufficientReasonCode = "duration_outlier" + cycle.Change.Projection.Assessment = &conclusion + cycle.Publish.SlackFloor = situationmodel.InterruptionHigh + commit := shDerive(t, cycle) + if err := st.CommitController(ctx, claim, commit); err != nil { + t.Fatal(err) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND status = 'withheld_by_operator_slack_floor'`, id); n != 1 { + t.Fatalf("fixture: withheld roots = %d, want 1", n) + } + return id +} + +// snCommitAboveFloor commits a second cycle for the withheld Situation that +// earns publication: an operator handoff ranks high, above the fixture's +// high floor (critical anchor conclusion). +func snCommitAboveFloor(t *testing.T, st *Store, id string, now time.Time) { + t.Helper() + ctx := context.Background() + shMakeDue(t, st, id, now.Add(-time.Minute)) + claim := claimSituation(t, st, id, "controller-a", now) + cycle := shPrepare(t, claim, shOperatorContract(now.Add(time.Minute)), + situationmodel.LifecycleActive, situationmodel.AttentionInvestigate, now) + var prior situationmodel.Transition + var summary situationmodel.EpisodeSummary + view, err := st.GetSituationEpisodeView(ctx, id) + if err != nil { + t.Fatal(err) + } + prior, summary = view.SourceTransition, view.Summary + cycle.Change.PriorTransition = &prior + cycle.Change.PriorSummary = &summary + cycle.Publish.PriorTransition = &prior + cycle.Publish.SlackFloor = situationmodel.InterruptionHigh + commit := shDerive(t, cycle) + if err := st.CommitController(ctx, claim, commit); err != nil { + t.Fatal(err) + } + if n := shCountRows(t, st, `SELECT COUNT(*) FROM notification_intents WHERE situation_id = ? AND effect_class = 'root_sync' AND status = 'pending'`, id); n != 1 { + t.Fatalf("fixture: pending roots after earning publication = %d, want 1", n) + } +} diff --git a/internal/store/notification_supersede_live_roots_upgrade_test.go b/internal/store/notification_supersede_live_roots_upgrade_test.go index 4a224fe..a63b110 100644 --- a/internal/store/notification_supersede_live_roots_upgrade_test.go +++ b/internal/store/notification_supersede_live_roots_upgrade_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" "time" + + situationmodel "github.com/alertint/alertint-agent/internal/situation/model" ) // ---------------------------------------------------------------------- @@ -117,12 +119,12 @@ func TestNotificationSupersedeLiveRootsUpgrade(t *testing.T) { if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - if got != 20 { - t.Fatalf("MaxSchemaVersion = %d, want 20", got) + if got != 21 { + t.Fatalf("MaxSchemaVersion = %d, want 21", got) } var version int - if err := st.db.QueryRowContext(ctx, `SELECT MAX(version) FROM schema_migrations`).Scan(&version); err != nil || version != 20 { - t.Fatalf("applied schema version = %d (err=%v), want 20", version, err) + if err := st.db.QueryRowContext(ctx, `SELECT MAX(version) FROM schema_migrations`).Scan(&version); err != nil || version != 21 { + t.Fatalf("applied schema version = %d (err=%v), want 21", version, err) } var fkViolations int if err := st.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pragma_foreign_key_check`).Scan(&fkViolations); err != nil || fkViolations != 0 { @@ -175,3 +177,49 @@ func supersessionTriggerNames(t *testing.T, st *Store) string { } return triggers } + +// TestNotificationUpgradeCoalescesOlderBlockedRootBeforeClaims pins review +// round 2, R2-F4: a schema-19 ledger could hold an older blocked root beside +// its newer pending projection (its trigger forbade retiring the former). +// After the upgrade, corrected configuration must coalesce the old blocked +// root into the pending keeper BEFORE the claim poll, or the blocked row +// holds the queue head forever. +func TestNotificationUpgradeCoalescesOlderBlockedRootBeforeClaims(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "review-upgrade.db") + blocked, _ := seedMigration19SupersedeFixture(t, path) + db, err := sql.Open("sqlite", buildDSN(path)) + if err != nil { + t.Fatal(err) + } + if _, err = db.ExecContext(ctx, `INSERT INTO notification_intents + (id, idempotency_key, effect_class, situation_id, transition_id, transition_sequence, summary_version, requires_root, + main_channel_poke, client_message_id, status, created_at) + SELECT 'root-new-pending', 'idem:new-pending', effect_class, situation_id, transition_id, transition_sequence, summary_version, + requires_root, main_channel_poke, 'client:new-pending', 'pending', created_at + FROM notification_intents WHERE id = ?`, blocked); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + st, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = st.Close() }() + now := time.Now().UTC().Add(time.Minute) + if _, err := st.ReactivateConfigurationBlocked(ctx, 1, now); err != nil { + t.Fatal(err) + } + claims, err := st.ClaimNotificationIntents(ctx, snOwner, now, time.Minute, 25) + if err != nil { + t.Fatal(err) + } + if len(claims) != 1 || claims[0].Intent.ID != "root-new-pending" { + t.Fatalf("upgraded schema-19 backlog still blocked after corrected configuration: got %d claims, want the newest pending root", len(claims)) + } + if older := snIntent(t, st, blocked); older.Status != situationmodel.IntentSuperseded { + t.Fatalf("pre-upgrade blocked root = %q, want superseded by the pending keeper", older.Status) + } +} From 48dd11117071a9463f3d3564cd30563018bdbf61 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 14:47:24 +0300 Subject: [PATCH 29/31] fix(store): index the live intents the claim poll ranks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widening the claim ranking to blocked and failed rows (round 1) left the poll's predicate unmatched by 0018's pending-only partial index, so every one-second poll scanned the whole append-only ledger before ranking — the unbounded growth 0019 was added to stop for the blocked-count read (review round 2, R2-F3). Migration 0021 adds a partial index over exactly the three live statuses, spelled as the poll's own predicate so SQLite's partial-index implication matches it verbatim; the ranking query is now a named constant and an EXPLAIN QUERY PLAN test pins that the poll searches notification_intents_live_idx and never scans the table. MaxSchemaVersion is 21. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- .../0021_notification_live_index.sql | 21 ++++++ ...notification_blocked_index_upgrade_test.go | 2 +- internal/store/situation_notifications.go | 55 +++++++++------- .../store/situation_notifications_test.go | 65 +++++++++++++++++++ internal/store/store_test.go | 2 +- 5 files changed, 119 insertions(+), 26 deletions(-) create mode 100644 internal/store/migrations/0021_notification_live_index.sql diff --git a/internal/store/migrations/0021_notification_live_index.sql b/internal/store/migrations/0021_notification_live_index.sql new file mode 100644 index 0000000..7aa87f0 --- /dev/null +++ b/internal/store/migrations/0021_notification_live_index.sql @@ -0,0 +1,21 @@ +-- SPDX-License-Identifier: FSL-1.1-ALv2 +-- +-- One partial index, no schema change: the notification worker's claim +-- poll ranks every LIVE intent — pending, blocked_configuration, failed — +-- per Situation (migration 0020 / review round 1 made blocked and failed +-- rows hold their Situation's queue head), and 0018's claim index covers +-- only `status = 'pending'`. Without this index the once-per-second poll +-- scanned the whole append-only ledger before ranking (review round 2, +-- R2-F3), the exact unbounded growth 0019 was added to stop for the +-- blocked-count read. +-- +-- The WHERE clause is spelled exactly as the claim query's predicate so +-- SQLite's partial-index implication matches it verbatim. Live rows are +-- bounded work (they resolve to delivered/superseded/withheld); resolved +-- history is excluded from the index and never read by the poll. +-- +-- This migration adds no table, no column, and no row: it fabricates +-- nothing for any Situation that predates it. +-- ---------------------------------------------------------------------- +CREATE INDEX notification_intents_live_idx ON notification_intents(status, situation_id, transition_sequence, id) + WHERE status IN ('pending', 'blocked_configuration', 'failed'); diff --git a/internal/store/notification_blocked_index_upgrade_test.go b/internal/store/notification_blocked_index_upgrade_test.go index 5cd74be..cd7d1b4 100644 --- a/internal/store/notification_blocked_index_upgrade_test.go +++ b/internal/store/notification_blocked_index_upgrade_test.go @@ -130,7 +130,7 @@ func TestNotificationBlockedIndexUpgrade_AddsThePartialIndexAndFabricatesNothing if err != nil { t.Fatalf("MaxSchemaVersion: %v", err) } - if got != 20 { + if got != 21 { t.Fatalf("MaxSchemaVersion = %d, want 19", got) } diff --git a/internal/store/situation_notifications.go b/internal/store/situation_notifications.go index eb1ec3d..cd003c6 100644 --- a/internal/store/situation_notifications.go +++ b/internal/store/situation_notifications.go @@ -249,6 +249,36 @@ func deliveryGapGateTx(ctx context.Context, tx *sql.Tx) (deliveryGapGate, error) return gate, nil } +// notificationClaimRankingQuery is the claim poll's ranking query: live +// rows only (the predicate is spelled exactly as migration 0021's partial +// index WHERE clause, so the poll never scans resolved history), one +// claimable head per Situation, in notificationClaimOrder. Bound values: +// now (lease), now (retry), limit. +const notificationClaimRankingQuery = ` + WITH ranked AS ( + SELECT ni.id AS id, + ni.gap_generation AS gap_generation, + ni.situation_id AS situation_id, + ni.transition_sequence AS transition_sequence, + ` + notificationRootFirst + ` AS root_first, + ` + notificationClassRank + ` AS class_rank, + (ni.status = 'pending') AS claimable, + (ni.claim_owner IS NULL OR ni.lease_expires_at <= ?) AS unleased, + (ni.retry_at IS NULL OR ni.retry_at <= ?) AS due, + (ni.requires_root = 0 OR (s.slack_channel IS NOT NULL AND s.slack_root_ts IS NOT NULL)) AS root_ready, + ROW_NUMBER() OVER ( + PARTITION BY ni.situation_id + ORDER BY ` + notificationRootFirst + ` ASC, ni.transition_sequence ASC, ` + notificationClassRank + ` ASC, ni.id ASC + ) AS rn + FROM notification_intents ni + LEFT JOIN situations s ON s.id = ni.situation_id + WHERE ni.status IN ('pending', 'blocked_configuration', 'failed') + ) + SELECT id FROM ranked + WHERE (situation_id IS NULL OR rn = 1) AND claimable AND unleased AND due AND root_ready + ` + notificationClaimOrder + ` + LIMIT ?` + // dueNotificationIntentIDsTx selects the ids this round may claim, in // notificationClaimOrder. func dueNotificationIntentIDsTx(ctx context.Context, tx *sql.Tx, gate deliveryGapGate, @@ -273,30 +303,7 @@ func dueNotificationIntentIDsTx(ctx context.Context, tx *sql.Tx, gate deliveryGa return ids, nil } - rows, err := tx.QueryContext(ctx, ` - WITH ranked AS ( - SELECT ni.id AS id, - ni.gap_generation AS gap_generation, - ni.situation_id AS situation_id, - ni.transition_sequence AS transition_sequence, - `+notificationRootFirst+` AS root_first, - `+notificationClassRank+` AS class_rank, - (ni.status = 'pending') AS claimable, - (ni.claim_owner IS NULL OR ni.lease_expires_at <= ?) AS unleased, - (ni.retry_at IS NULL OR ni.retry_at <= ?) AS due, - (ni.requires_root = 0 OR (s.slack_channel IS NOT NULL AND s.slack_root_ts IS NOT NULL)) AS root_ready, - ROW_NUMBER() OVER ( - PARTITION BY ni.situation_id - ORDER BY `+notificationRootFirst+` ASC, ni.transition_sequence ASC, `+notificationClassRank+` ASC, ni.id ASC - ) AS rn - FROM notification_intents ni - LEFT JOIN situations s ON s.id = ni.situation_id - WHERE ni.status IN ('pending', 'blocked_configuration', 'failed') - ) - SELECT id FROM ranked - WHERE (situation_id IS NULL OR rn = 1) AND claimable AND unleased AND due AND root_ready - `+notificationClaimOrder+` - LIMIT ?`, nowStr, nowStr, limit) + rows, err := tx.QueryContext(ctx, notificationClaimRankingQuery, nowStr, nowStr, limit) if err != nil { return nil, fmt.Errorf("store: select due notification intents: %w", err) } diff --git a/internal/store/situation_notifications_test.go b/internal/store/situation_notifications_test.go index e08e32a..6996c8b 100644 --- a/internal/store/situation_notifications_test.go +++ b/internal/store/situation_notifications_test.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "strings" "testing" "time" @@ -1246,3 +1247,67 @@ func snDump(t *testing.T, st *Store) string { } return out.String() } + +// ---------------------------------------------------------------------- +// The claim poll never scans resolved history (review round 2, R2-F3). +// ---------------------------------------------------------------------- + +func TestNotificationClaimQueryUsesTheLiveIndex(t *testing.T) { + st := newTestStore(t) + rows, err := st.db.QueryContext(context.Background(), `EXPLAIN QUERY PLAN `+notificationClaimRankingQuery, + "2026-09-06T10:00:00Z", "2026-09-06T10:00:00Z", 25) + if err != nil { + t.Fatal(err) + } + defer func() { _ = rows.Close() }() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + if strings.Contains(detail, "SCAN ni") { + t.Errorf("claim poll scans notification history rather than an index restricted to live statuses: %s", detail) + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(plan, "\n"), "notification_intents_live_idx") { + t.Fatalf("claim poll does not use notification_intents_live_idx:\n%s", strings.Join(plan, "\n")) + } +} + +// ---------------------------------------------------------------------- +// Reactivation coalesces older live roots whatever the keeper's status +// (review round 2, R2-F4). +// ---------------------------------------------------------------------- + +func TestNotificationReactivationRetiresAnOlderBlockedRootBehindADeliveredOne(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "reactivate-delivered.db") + blockedRootID, deliveredRootID := seedMigration19SupersedeFixture(t, path) + st, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = st.Close() }() + + n, err := st.ReactivateConfigurationBlocked(ctx, 1, time.Now().UTC().Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("reactivated %d root projections, want 0: the delivered newer projection is the root on screen", n) + } + older := snIntent(t, st, blockedRootID) + if older.Status != situationmodel.IntentSuperseded || older.ReplacementIntentID == nil || *older.ReplacementIntentID != deliveredRootID { + t.Fatalf("pre-upgrade blocked root after reactivation = %q (replacement %v), want superseded by the delivered %s: an obsolete root must never be revived", + older.Status, older.ReplacementIntentID, deliveredRootID) + } + if got := snIntent(t, st, deliveredRootID); got.Status != situationmodel.IntentDelivered || got.MessageTS == nil { + t.Fatalf("delivered root after reactivation = %+v, want its delivered outcome untouched", got) + } +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 8cc17ae..7327464 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -451,7 +451,7 @@ func TestMaxSchemaVersion(t *testing.T) { // migration ownership") plus 0019, which adds one partial index and no // schema of its own — 0017 and 0018 are final and are never edited, so a // new migration is the only sanctioned way to add it. - if got != 20 { + if got != 21 { t.Errorf("MaxSchemaVersion = %d, want 19", got) } } From c1e5b6765192ea95f9a3f60d8927d1c4d16fdfc0 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 15:11:41 +0300 Subject: [PATCH 30/31] fix(situation): internal work progress is never a changed-required-action poke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PokeRequiredActionChanged compared the whole Operator contract without its deadline, which includes AlertINT's work action/status, wait reason, and update triggers. Once the repage cooldown had elapsed, Triage starting or finishing produced a new broadcast_handoff for an unchanged investigate_situation action — a main-channel interruption outside the spec's closed permitted-poke list (review round 3, R3-F1). The class now requires the requested human action itself to differ between the prior Transition and this one, the same basis HandoffStillCurrent uses, so an internal contract change edits the root and journals its material entry without ever poking. With this build's one-entry operator-action catalog the class is reserved rather than reachable; its cooldown gate is pinned on the helper directly, and the production-derived Triage start/finish cases are pinned as never poking. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- internal/situation/notification_plan_test.go | 96 +++++++++++++++----- internal/situation/priority.go | 27 +++--- internal/situation/priority_test.go | 41 ++++++++- 3 files changed, 127 insertions(+), 37 deletions(-) diff --git a/internal/situation/notification_plan_test.go b/internal/situation/notification_plan_test.go index eb01e05..754b787 100644 --- a/internal/situation/notification_plan_test.go +++ b/internal/situation/notification_plan_test.go @@ -615,30 +615,46 @@ func TestPlanNotificationIntentsRepageCooldown(t *testing.T) { return c } - t.Run("a changed required action inside the cooldown does not repage", func(t *testing.T) { - c := changeAt(2*time.Minute, nil) - trs, folded := hsCommitOf(t, c) - in := hsPub(c, trs, folded) - in.LastMainChannelPokeAt = timePtr(handedOff.CreatedAt) + // An internal contract change (a different reconsideration trigger, + // same human action) is material and journaled but never a poke — + // inside OR outside the cooldown (review round 3, R3-F1): the cooldown + // gates a changed required action, and its expiry grants nothing. + for name, offset := range map[string]time.Duration{"inside the cooldown": 2 * time.Minute, "after the cooldown": 20 * time.Minute} { + t.Run("internal work progress never repages "+name, func(t *testing.T) { + c := changeAt(offset, nil) + trs, folded := hsCommitOf(t, c) + in := hsPub(c, trs, folded) + in.LastMainChannelPokeAt = timePtr(handedOff.CreatedAt) + + got := hsPlan(t, in) + if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { + t.Errorf("got %d broadcasts, want 0: unchanged human action", len(broadcasts)) + } + if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { + t.Errorf("the journal entry is never cooled down, got %d", len(threads)) + } + if roots := hsIntentsOfClass(got, model.EffectRootSync); len(roots) != 1 { + t.Errorf("internal progress still edits the root, got %d", len(roots)) + } + }) + } - got := hsPlan(t, in) - if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 0 { - t.Errorf("got %d broadcasts inside the cooldown, want 0", len(broadcasts)) + // The cooldown gate itself, tested on its helper: the reserved + // changed-action class is the only one it applies to and no Transition + // this build can produce reaches it. + t.Run("the cooldown helper", func(t *testing.T) { + in := hsPub(hsChange(t), nil, model.EpisodeSummary{}) + in.LastMainChannelPokeAt = timePtr(in.Now.Add(-2 * time.Minute)) + if cooldownElapsed(in) { + t.Error("2 minutes after the last poke is inside a 15-minute cooldown") } - if threads := hsIntentsOfClass(got, model.EffectThreadAppend); len(threads) != 1 { - t.Errorf("the journal entry is never cooled down, got %d", len(threads)) + in.LastMainChannelPokeAt = timePtr(in.Now.Add(-20 * time.Minute)) + if !cooldownElapsed(in) { + t.Error("20 minutes after the last poke is past a 15-minute cooldown") } - }) - - t.Run("a changed required action after the cooldown repages", func(t *testing.T) { - c := changeAt(20*time.Minute, nil) - trs, folded := hsCommitOf(t, c) - in := hsPub(c, trs, folded) - in.LastMainChannelPokeAt = timePtr(handedOff.CreatedAt) - - got := hsPlan(t, in) - if broadcasts := hsIntentsOfClass(got, model.EffectBroadcastHandoff); len(broadcasts) != 1 { - t.Errorf("got %d broadcasts after the cooldown, want 1", len(broadcasts)) + in.LastMainChannelPokeAt = nil + if !cooldownElapsed(in) { + t.Error("no prior poke means no cooldown") } }) @@ -1099,3 +1115,41 @@ func TestPlanNotificationIntentsRecurrenceModeChangeGatedPostsAQuietMilestone(t t.Fatalf("a recurrence milestone must never re-page the channel, got %+v", broadcasts) } } + +// TestPlanNotificationIntentsInternalWorkProgressNeverRepages is review +// round 3's reproduction, kept verbatim in intent: production-derived +// contracts for Triage starting and finishing, same human action, a poke +// 20 minutes ago — the planner edits the root and journals, and creates no +// main-channel poke. +func TestPlanNotificationIntentsInternalWorkProgressNeverRepages(t *testing.T) { + for name, phase := range map[string]TriagePhase{"triage_started": TriagePhaseInFlight, "triage_finished": TriagePhaseNone} { + t.Run(name, func(t *testing.T) { + c := hsNext(t) + hsUseReason(&c, reasonCodeDurationOutlier) + action := model.OperatorActionInvestigateSituation + state := ControllerState{Lifecycle: model.LifecycleActive, Attention: model.AttentionInvestigate, + OperatorActionRequired: &action, TriagePhase: TriagePhaseAwaitingDecision} + c.PriorTransition.ActionContract = DeriveActionContract(state, DeriveCadence(state), c.Now.Add(-20*time.Minute)) + c.PriorTransition.Projection = c.Projection + c.PriorSummary.ActionContract = c.PriorTransition.ActionContract + state.TriagePhase = phase + c.Assessment.ActionContract = DeriveActionContract(state, DeriveCadence(state), c.Now) + transitions, summary := hsCommitOf(t, c) + if len(transitions) == 0 { + t.Fatal("internal progress should still update the root/history") + } + in := hsPub(c, transitions, summary) + lastPoke := c.Now.Add(-20 * time.Minute) + in.LastMainChannelPokeAt = &lastPoke + intents := hsPlan(t, in) + if n := len(hsIntentsOfClass(intents, model.EffectRootSync)); n != 1 { + t.Fatalf("got %d root edits, want 1", n) + } + for _, intent := range intents { + if intent.MainChannelPoke { + t.Errorf("unchanged investigate_situation action plus internal Triage progress created a %s channel poke; cooldown expiry alone grants no publication authority", intent.EffectClass) + } + } + }) + } +} diff --git a/internal/situation/priority.go b/internal/situation/priority.go index 11570c8..6741829 100644 --- a/internal/situation/priority.go +++ b/internal/situation/priority.go @@ -112,8 +112,16 @@ const ( PokeUrgentAttention PokeClass = "urgent_attention" // PokeOperatorHandoff is a no-action to operator-judgment/action handoff. PokeOperatorHandoff PokeClass = "operator_handoff" - // PokeRequiredActionChanged is a materially changed required action; it - // is the one class the configured repage cooldown gates. + // PokeRequiredActionChanged is a materially changed required HUMAN + // action — the operator is now asked for something different; it is the + // one class the configured repage cooldown gates. AlertINT's own work + // progress (Triage starting or finishing, a wait reason, an update + // trigger) changes the Operator contract but not what the operator is + // asked to do, and is never this class: a cooldown restricts a changed + // action, its expiry does not turn an unchanged one into a poke (review + // round 3, R3-F1). With this build's one-entry operator-action catalog + // the class is reserved: reachable once a second supported action + // exists, never by broadening authority. PokeRequiredActionChanged PokeClass = "required_action_changed" ) @@ -157,8 +165,8 @@ func ClassifyPoke(prior *model.Transition, t model.Transition) PokeClass { return PokeUrgentAttention case t.ActionContract.OperatorActionRequired != nil && prior.ActionContract.OperatorActionRequired == nil: return PokeOperatorHandoff - case t.ActionContract.OperatorActionRequired != nil && - operatorContractTuple(prior.ActionContract) != operatorContractTuple(t.ActionContract): + case t.ActionContract.OperatorActionRequired != nil && prior.ActionContract.OperatorActionRequired != nil && + derefOperatorAction(t.ActionContract.OperatorActionRequired) != derefOperatorAction(prior.ActionContract.OperatorActionRequired): return PokeRequiredActionChanged default: return PokeNone @@ -186,15 +194,8 @@ func ClassifyPoke(prior *model.Transition, t model.Transition) PokeClass { // Attention, newly crossed criticality) is current while its Attention // still holds. // -// This is deliberately narrower than PokeRequiredActionChanged's basis -// (the whole contract without its deadline): that class decides whether a -// NEW interruption is warranted, this function decides whether an -// interruption already owed may be demoted. With the current one-entry -// operator-action catalog the two can disagree on an internal contract -// change — the queued handoff still broadcasts as current, and the newer -// Transition may broadcast again once the repage cooldown allows — which is -// the side to err on: the spec demotes only when the requested action is -// no longer current. +// PokeRequiredActionChanged is judged on the same basis, so an internal +// contract change neither demotes an owed handoff nor earns a new poke. // // A summary that does not yet include the handoff cannot confirm it and // counts as not current. diff --git a/internal/situation/priority_test.go b/internal/situation/priority_test.go index f9b6435..fc9534e 100644 --- a/internal/situation/priority_test.go +++ b/internal/situation/priority_test.go @@ -207,6 +207,10 @@ func TestInterruptionPriorityRequiredActionChangeIsCooldownGated(t *testing.T) { t.Fatalf("ProjectEpisode: %v", err) } + // AlertINT's own machinery moves (a different reconsideration trigger) + // while the operator is asked for exactly the same thing: material, + // journaled, but never a poke — a cooldown restricts a changed action, + // it does not turn an unchanged one into one (review round 3, R3-F1). changed := hsChange(t) changed.Now = handedOff.CreatedAt.Add(time.Minute) changed.Situation.InputVersion = 9 @@ -216,10 +220,16 @@ func TestInterruptionPriorityRequiredActionChangeIsCooldownGated(t *testing.T) { contract.NextUpdateOn = []model.NextUpdateOn{model.NextUpdateOnSourceResolution} changed.Assessment.ActionContract = contract next := hsOnly(t, changed) - - if got := ClassifyPoke(&handedOff, next); got != PokeRequiredActionChanged { - t.Fatalf("poke class = %q, want %q", got, PokeRequiredActionChanged) + if next.Reason != model.ReasonOperatorContractChanged { + t.Fatalf("fixture: reason = %q, want operator_contract_changed (still material)", next.Reason) + } + if got := ClassifyPoke(&handedOff, next); got != PokeNone { + t.Fatalf("poke class for internal work progress = %q, want %q", got, PokeNone) } + + // The reserved class itself: gated by the cooldown, unlike every + // escalation class. Reachable only once a second supported operator + // action exists (the catalog has one today). if !PokeRequiredActionChanged.CooldownApplies() { t.Error("a materially changed required action must respect the repage cooldown") } @@ -230,6 +240,31 @@ func TestInterruptionPriorityRequiredActionChangeIsCooldownGated(t *testing.T) { } } +// TestInterruptionPriorityInternalWorkProgressNeverPokes pins review round +// 3, R3-F1, on contracts derived through production DeriveActionContract: +// Triage starting and Triage finishing keep investigate_situation +// outstanding and are not a changed required action. +func TestInterruptionPriorityInternalWorkProgressNeverPokes(t *testing.T) { + for name, phase := range map[string]TriagePhase{"triage_started": TriagePhaseInFlight, "triage_finished": TriagePhaseNone} { + t.Run(name, func(t *testing.T) { + c := hsNext(t) + hsUseReason(&c, reasonCodeDurationOutlier) + action := model.OperatorActionInvestigateSituation + state := ControllerState{Lifecycle: model.LifecycleActive, Attention: model.AttentionInvestigate, + OperatorActionRequired: &action, TriagePhase: TriagePhaseAwaitingDecision} + c.PriorTransition.ActionContract = DeriveActionContract(state, DeriveCadence(state), c.Now.Add(-20*time.Minute)) + c.PriorTransition.Projection = c.Projection + c.PriorSummary.ActionContract = c.PriorTransition.ActionContract + state.TriagePhase = phase + c.Assessment.ActionContract = DeriveActionContract(state, DeriveCadence(state), c.Now) + next := hsOnly(t, c) + if got := ClassifyPoke(c.PriorTransition, next); got != PokeNone { + t.Fatalf("poke class = %q, want %q: unchanged investigate_situation plus internal progress", got, PokeNone) + } + }) + } +} + func TestInterruptionPriorityFloorComparison(t *testing.T) { cases := []struct { priority, floor model.InterruptionPriority From 76949406c168b1e4cc08288ea5dfa670d4efe6a0 Mon Sep 17 00:00:00 2001 From: ernescz Date: Sun, 6 Sep 2026 16:48:07 +0300 Subject: [PATCH 31/31] fix(situation): a concluded investigation marks the episode as investigated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lab 2026-09-06 run C (Plan 3 Slack-enabled acceptance): a Situation published from the deterministic critical floor while L2 was unreachable carried retry_situation_assessment in its first contract. Because that contract already names investigation work, no later Transition was ever classified investigation_started; Acute Triage then ran and investigation_concluded was journaled, but EpisodeSummary.InvestigationStarted stayed false and the root's orientation fell from Investigating back to Observed at the conclusion, contradicting the thread it sat above. ProjectEpisode now sets InvestigationStarted when it folds an investigation_concluded Transition — a concluded investigation necessarily started. A contract that merely names investigation work still does not set the flag: a clean Triage skip and a direct closed_unknown keep reading as no investigation having run (existing tests pin both). Summaries persisted before this change keep their recorded flag; the lab record notes run C's root as pre-fix evidence. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019V7y36hYk1saPdcd5fr9hg Signed-off-by: ernescz --- internal/situation/history.go | 14 ++++++++++ internal/situation/history_test.go | 43 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/internal/situation/history.go b/internal/situation/history.go index 9b6cc77..805e16f 100644 --- a/internal/situation/history.go +++ b/internal/situation/history.go @@ -853,6 +853,20 @@ func ProjectEpisode(prior *model.EpisodeSummary, t model.Transition) (model.Epis out.ImpactSummary = impactSummary(concl.Impact) } + // A concluded investigation necessarily started. A first authoritative + // state whose fallback contract is already retry_situation_assessment + // (an L2 outage at publication) never yields a later Transition + // classified investigation_started, yet Triage runs and + // investigation_concluded is journaled under it; without this the + // summary flag stayed false and the root fell back from Investigating + // to Observed at the conclusion (lab 2026-09-06, run C). A contract + // that merely names investigation work does NOT set the flag here: a + // clean Triage skip and a direct closed_unknown must never read as an + // investigation having run. + if t.Reason == model.ReasonInvestigationConcluded { + out.InvestigationStarted = true + } + switch t.JournalKind { //nolint:exhaustive // only the accumulating journal kinds contribute to the two bounded summary lists; every other kind updates the scalar fields above. case model.JournalInvestigationStarted: out.InvestigationStarted = true diff --git a/internal/situation/history_test.go b/internal/situation/history_test.go index 3b9c2da..778e553 100644 --- a/internal/situation/history_test.go +++ b/internal/situation/history_test.go @@ -1540,3 +1540,46 @@ func TestProjectEpisodeFoldsTheRestOfOneTerminalCommitButNeverReopens(t *testing t.Fatal("a later commit reusing the same terminal instant must never fold onto a terminal Episode") } } + +// Lab 2026-09-06 run C: a Situation published from the deterministic critical +// floor while L2 was unreachable carried retry_situation_assessment in its +// FIRST contract, so no later Transition could be classified +// investigation_started; Triage then ran and investigation_concluded was +// journaled, but the summary flag stayed false and the root fell from +// Investigating back to Observed at the conclusion. A concluded +// investigation necessarily started. (A contract that merely names +// investigation work does not set the flag — see the clean-skip and direct +// closed_unknown tests above.) +func TestProjectEpisodeInvestigationConcludedMarksTheInvestigationStarted(t *testing.T) { + first := hsFirst(t) // baseline contract: run_acute_triage running, no start Transition + if first.Reason != model.ReasonFirstAuthoritativeState { + t.Fatalf("first reason = %q", first.Reason) + } + sum, err := ProjectEpisode(nil, first) + if err != nil { + t.Fatalf("ProjectEpisode(first): %v", err) + } + if sum.InvestigationStarted { + t.Fatal("a contract that names investigation work must not by itself mark the investigation started") + } + if got := DeriveOrientation(sum, first); got != OrientationInvestigating { + t.Fatalf("orientation while the contract investigates = %q, want %q", got, OrientationInvestigating) + } + + c := hsNext(t) + c.Assessment.ActionContract = hsMonitoringContract(c.Now.Add(time.Minute)) + concluded := hsOnly(t, c) + if concluded.Reason != model.ReasonInvestigationConcluded { + t.Fatalf("reason = %q, want investigation_concluded", concluded.Reason) + } + sum, err = ProjectEpisode(&sum, concluded) + if err != nil { + t.Fatalf("ProjectEpisode(concluded): %v", err) + } + if !sum.InvestigationStarted { + t.Fatal("investigation_concluded must leave InvestigationStarted true") + } + if got := DeriveOrientation(sum, concluded); got != OrientationInvestigating { + t.Fatalf("orientation after conclusion = %q, want %q", got, OrientationInvestigating) + } +}