Plan 3: Situation history and Slack ownership

README.md

@@ -14,7 +14,7 @@
 
 > AlertINT turns infrastructure alerts into investigated incidents and serves them to the AI tools you already use, over MCP — a self-hosted agent that runs inside your own network.
 
-A single Go binary that sits between your monitoring stack and your AI agent. It ingests alert webhooks from Alertmanager and Zabbix, correlates them into incidents through an open rule engine, and runs an LLM triage that falsifies its own draft verdict before the finding ships. Findings go to Slack; the incident state — plus read-only Prometheus, Loki, and Zabbix access — is exposed to any MCP client. Corrections your agent captures over MCP steer the next triage of the same failure. Read-only by design. Local state. You bring the LLM key.
+A single Go binary that sits between your monitoring stack and your AI agent. It ingests alert webhooks from Alertmanager and Zabbix, correlates them into incidents through an open rule engine, and runs an LLM triage that falsifies its own draft verdict before the finding ships. Findings go to stdout and, when configured, to one Slack channel; the incident state — plus read-only Prometheus, Loki, and Zabbix access — is exposed to any MCP client. Corrections your agent captures over MCP steer the next triage of the same failure. Read-only by design. Local state. You bring the LLM key.
 
 **Full documentation: [alertint.com/docs](https://alertint.com/docs)**
 
@@ -57,6 +57,16 @@ The whole pipeline — receivers, correlation, the evidence pack, both loops, an
 the MCP surface — is diagrammed and walked through step by step in
 **[Architecture](https://alertint.com/docs/concepts/architecture)**.
 
+On the `state-controller` integration branch (not the released default), a
+durable **Situation** owns each failure group's history: every authoritative
+material change commits one immutable transition and one version of a current
+episode summary, and Slack shows one evolving Situation root plus an
+immutable ordered journal thread instead of a per-incident card. Delivery is
+driven from durable intents that retry indefinitely, open a visible gap after
+five continuous minutes of Slack failure, and replay every affected episode
+in order once Slack returns — see
+**[Slack](https://alertint.com/docs/notifications/slack)**.
+
 ## Documentation
 
 - **[Docs home](https://alertint.com/docs)** — quickstart, configuration reference

cmd/alertint/drill.go

@@ -249,7 +249,7 @@ func (d *drillCmd) run(ctx context.Context) error {
 				return err
 			}
 		} else {
-			d.printf("fired the rerun; mcp is not usable from here — check the DRILL card edit to \"recurred ×N\".")
+			d.printf("fired the rerun; mcp is not usable from here — the occurrence count is visible over mcp or in the incidents table, and the owning Situation's root shows it as recurred ×N once it next edits.")
 		}
 		return d.maybeResolve(ctx, run, recvBase, webhookToken)
 	}
@@ -499,7 +499,10 @@ func (d *drillCmd) fetchDrillCandidates(ctx context.Context, mcpEndpoint, mcpTok
 
 // pollOccurrenceRerun polls the matched incident until its occurrence count
 // registers the collapsed re-fire, then prints the "recurred ×N" payoff. It
-// exits as soon as the count increments; a timeout points at the card edit.
+// exits as soon as the count increments. The attach feeds the owning
+// Situation's recurrence count; only a crossed milestone rung creates a
+// Transition (and one quiet thread reply), so a single rerun usually shows
+// nothing new in Slack until the root next edits.
 func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToken, incidentID string) error {
 	client := newMCPOneShotClient(mcpEndpoint, mcpToken, d.http)
 	if err := client.initialize(ctx); err != nil {
@@ -518,7 +521,7 @@ func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToke
 				Occurrences int `json:"occurrences"`
 			}
 			if json.Unmarshal(raw, &p) == nil && p.Occurrences > 0 {
-				d.printf("collapsed: incident %s recurred ×%d — no second triage, the existing card edits in place", incidentID, p.Occurrences+1)
+				d.printf("collapsed: incident %s recurred ×%d — no second triage; the owning Situation's recurrence count moved", incidentID, p.Occurrences+1)
 				return nil
 			}
 		}
@@ -528,7 +531,7 @@ func (d *drillCmd) pollOccurrenceRerun(ctx context.Context, mcpEndpoint, mcpToke
 			}
 		}
 	}
-	d.printf("the occurrence has not registered yet; check the DRILL card edit to \"recurred ×N\", or re-run with --result %s", incidentID)
+	d.printf("the occurrence has not registered yet; re-run with --result %s (the attach shows up as the owning Situation's recurrence count, not as a card edit of its own)", incidentID)
 	return nil
 }
 

cmd/alertint/main.go

@@ -51,7 +51,6 @@ import (
 	"github.com/alertint/alertint-agent/internal/logs/loki"
 	internalmcp "github.com/alertint/alertint-agent/internal/mcp"
 	"github.com/alertint/alertint-agent/internal/notify"
-	notifyresolution "github.com/alertint/alertint-agent/internal/notify/resolution"
 	notifyslack "github.com/alertint/alertint-agent/internal/notify/slack"
 	notifystdout "github.com/alertint/alertint-agent/internal/notify/stdout"
 	promclient "github.com/alertint/alertint-agent/internal/prometheus"
@@ -389,17 +388,29 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error {
 	}
 	cor := correlator.New(corCfg, st, productionIncidentSink(), logger)
 
-	// SetTriageFailureNotifier is safe to wire here, before reconstruction:
-	// it is not reachable from ApplyDelivery's durable-dispatch path (the
-	// triage-exhausted notifier fires only from the correlator's own
-	// internal ticker loop, which isn't running during reconstruction).
+	// Task 8: production wires NO notifier onto the Correlator at all any
+	// more — not SetTriageFailureNotifier here, and not
+	// SetResolutionNotifier/SetOccurrenceNotifier inside startCorrelator
+	// below either. The Situation notification worker (Task 6/7) is now the
+	// sole production Slack writer; the Correlator's own
+	// ResolutionNotifier/OccurrenceNotifier/TriageFailureNotifier setters
+	// stay real Go shape (this package's own tests, and
+	// internal/correlator's, keep exercising them with fakes) but are never
+	// handed a live instance here. Each domain outcome's durable Situation
+	// input (incident_resolved, membership_changed for an occurrence
+	// attach, triage_exhausted) is written directly by domain logic inside
+	// the relevant atomic store commit — see correlator.go's own doc
+	// comments on the three interfaces — so leaving all three unwired loses
+	// no durable history, only the retired Incident-card Slack/stdout
+	// fan-out.
+	//
 	// The Correlator has no analyzer/LLM seam at all — no IncidentSink
 	// beyond the no-op one and no re-judgment runner (Plan 2 Task 7) —
 	// see TestProductionCorrelatorHasNoAcuteTriageDispatchDependency.
-	// SetAuditor, SetResolutionNotifier, and SetOccurrenceNotifier are NOT
-	// wired here — all three ARE reachable from ApplyDelivery — see
-	// startCorrelator below.
-	cor.SetTriageFailureNotifier(notifier)
+	// SetAuditor IS still wired — inside startCorrelator below, alongside
+	// this comment's former SetResolutionNotifier/SetOccurrenceNotifier
+	// neighbors — because it alone remains reachable from ApplyDelivery's
+	// durable-dispatch path.
 
 	// stopCorrelator is called exactly once, however runServe exits: inline,
 	// in the right relative position, by foundationStopSequence on the
@@ -440,11 +451,22 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error {
 	// the Correlator itself (cor, constructed above) receives no LLM
 	// dependency of its own — corCfg/correlator.New's signature carries none,
 	// and its only path to Acute Triage is via incidentSink{skill: skill}.
-	crt, err := buildControllerRuntime(st, llmClient, llmHealth, skill, cfg.Situations, owner, auditor, logger)
+	crt, err := buildControllerRuntime(st, llmClient, llmHealth, skill, cfg.Situations,
+		cfg.Notify.Slack.MinSeverity, cfg.Notify.Slack.RecurrenceMode, owner, auditor, logger)
 	if err != nil {
 		return err
 	}
 
+	// The Situation notification runtime (Plan 3 Task 9): the single
+	// reachable Situation Slack writer (present only when Situation Slack is
+	// actually configured — buildSituationNotificationRuntime) plus the
+	// stdout Transition-stream worker, which always runs because the
+	// authoritative outward state stream is not Slack-gated. Its own
+	// startup-only recovery pass (spec.md startup steps 2-6) runs after
+	// Plan 2's controller recovery and before the Correlator, and both its
+	// workers stop LAST, outside the shutdown drain rounds (R6).
+	nrt := buildSituationNotificationRuntime(cfg, st, auditor, owner, logger)
+
 	// Probe enabled integrations in the background: quickly (with backoff)
 	// while one is failing — at startup a co-deployed dependency may still
 	// be booting — then at a steady pace, logging losses and recoveries.
@@ -466,29 +488,40 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error {
 		backfillAndRecoverControllerWork: func(ctx context.Context) error {
 			return runControllerRecovery(ctx, crt, logger)
 		},
-		// SetAuditor/SetResolutionNotifier/SetOccurrenceNotifier are wired
-		// here — between reconstruct and cor.Start, never before — because
-		// all three are synchronously reachable from ApplyDelivery's
-		// durable-dispatch path (a queued resolved delivery whose commit
-		// settles an Incident calls the resolution notifier; a queued firing
-		// delivery that collapses into a recurrence occurrence calls the
-		// occurrence notifier and appends the occurrence_attached audit
-		// event; a queued retry attach appends triage_member_attached).
-		// Wiring them before reconstruction would let a plain
-		// crash-and-restart with ordinary queued webhook traffic post to
-		// Slack — or append audit rows — from reconstruction: exactly the
-		// outward effects the spec's "reconstruction invokes no notifier,
-		// audit callback, ..." acceptance forbids. The Setters' own doc
-		// comments require only "after New, before Start", so this ordering
-		// is legal; the Correlator's loop itself isn't running yet either.
+		// Plan 3 Task 9: recover abandoned notification/stream claims,
+		// schedule Situations whose durable history is missing or whose root
+		// projection is stale, validate the Slack configuration and record
+		// its generation, reactivate configuration-blocked effects, and
+		// resume an interrupted gap replay — all startup-only and all
+		// publication-free, exactly like the two recovery passes above.
+		recoverNotificationWork: func(ctx context.Context) error {
+			return runNotificationRecovery(ctx, nrt, logger)
+		},
+		// SetAuditor is wired here — between reconstruct and cor.Start, never
+		// before — because it is synchronously reachable from ApplyDelivery's
+		// durable-dispatch path (a queued retry attach appends
+		// triage_member_attached; a queued firing delivery that collapses
+		// into a recurrence occurrence appends occurrence_attached).
+		// Wiring it before reconstruction would let a plain crash-and-restart
+		// with ordinary queued webhook traffic append audit rows from
+		// reconstruction: exactly the outward effect the spec's
+		// "reconstruction invokes no notifier, audit callback, ..."
+		// acceptance forbids. The Setter's own doc comment requires only
+		// "after New, before Start", so this ordering is legal; the
+		// Correlator's loop itself isn't running yet either.
+		//
+		// Task 8: SetResolutionNotifier/SetOccurrenceNotifier are NOT called
+		// here (or anywhere in production) any more — see the comment above
+		// cor's construction. Their durable Situation inputs are written
+		// directly by ApplyCorrelatedDelivery, in the same atomic commit
+		// this dispatch path already runs, independent of any notifier.
 		startCorrelator: func(ctx context.Context) error {
 			cor.SetAuditor(auditor)
-			cor.SetResolutionNotifier(notifyresolution.New(notifier, st))
-			cor.SetOccurrenceNotifier(notifier)
 			return cor.Start(ctx)
 		},
-		startWorkers:           rt.Start,
-		startControllerWorkers: crt.Start,
+		startWorkers:             rt.Start,
+		startControllerWorkers:   crt.Start,
+		startNotificationWorkers: nrt.Start,
 		startReceivers: func() error {
 			var err error
 			recvSrv, recvErrCh, err = startReceivers(cfg, st, auditor, healthReg, llmHealth, rt.WakeDispatch, logger)
@@ -539,6 +572,10 @@ func runServe(args []string, _ io.Writer, stderr io.Writer) error {
 		drainControllerWork:   crt.Drain,
 		stopControllerWorkers: crt.Stop,
 		stopWorkers:           rt.Stop,
+		// R6, last and outside the drain rounds: one bounded final delivery
+		// and stdout pass under the shutdown context, then claim release. An
+		// unreachable Slack delays the pass, it never holds the process.
+		stopNotificationWorkers: nrt.Stop,
 	}
 	if err := stopSeq.run(shutdownCtx); err != nil {
 		logger.Error("situation foundation shutdown failed", slog.String("err", err.Error()))
@@ -1110,11 +1147,14 @@ func buildHealthChecks(cfg *config.Config, prom *promclient.Client, logSrc logs.
 //   - stdout: always an active sink when notify.stdout is set, so a send is
 //     confirmed (notified · stdout=ok) at INFO. Its verbose full JSON line is
 //     written only at debug level (consistently, in every format).
-//   - slack: when enabled and a bot token resolves.
 //
-// buildNotifier also returns the llmhealth.Publisher for the installation's
-// one system-message surface: the same Slack *Notifier when Slack is wired,
-// else nil (LLM dependency health then lives in state/audit/logs only).
+// Task 8: Slack is never registered into this Incident fan-out any more —
+// the Situation notification worker (Task 6/7) is the sole production Slack
+// writer for anything Incident-shaped now (findings, resolutions, occurrence
+// attaches, annotations/Captured verdicts). buildNotifier still constructs
+// the concrete Slack *Notifier and returns it as the llmhealth.Publisher
+// below when Slack is enabled and its bot token resolves — that remaining
+// production Slack use (ADR-0042/0046 System messages) is unaffected.
 func buildNotifier(cfg *config.Config, st *store.Store, auditor *audit.Auditor, logger *slog.Logger, debug bool) (*notify.Multi, llmhealth.Publisher) {
 	var nn []notify.Notifier
 	var sinks []string
@@ -1126,9 +1166,11 @@ func buildNotifier(cfg *config.Config, st *store.Store, auditor *audit.Auditor,
 	}
 	if cfg.Notify.Slack.Enabled {
 		if token, err := cfg.SlackBotToken(); err == nil && token != "" {
+			// Constructed for the System-message surface only
+			// (ADR-0042/0046, the llmhealth.Publisher return below) — never
+			// appended to nn: an Incident Slack card or thread reply is the
+			// Situation notification worker's job now, not this fan-out's.
 			slackNotifier := notifyslack.New(token, cfg.Notify.Slack.Channel, cfg.Notify.Slack.MinSeverity, cfg.Notify.Slack.RecurrenceMode, st, auditor)
-			nn = append(nn, slackNotifier)
-			sinks = append(sinks, "slack")
 			slackWired = true
 			publisher = slackNotifier
 		}